From 45282b23a1f21be89afdd2ad3779be7330523cef Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Wed, 21 May 2025 22:11:29 -0400 Subject: [PATCH 01/26] orbit: add automated feedback based on gradable status Use the last character of the status string to determine if the patchset was accepted, rejected, or there are not fatal issues such as whitespace errors, but don't reveal the full status string Signed-off-by: Joel Savitz --- mailman/patchset.py | 2 +- orbit/radius.py | 35 +++++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/mailman/patchset.py b/mailman/patchset.py index 7738f05d..8ffac792 100644 --- a/mailman/patchset.py +++ b/mailman/patchset.py @@ -80,7 +80,7 @@ def do_git_am(extra_args=[]): return f'patch {i+1} failed to apply!' if whitespace_errors: - return ('whitespace error patch(es) ' + return (f'whitespace error patch{"es" if len(whitespace_errors) > 1 else ""} ' f'{",".join(whitespace_errors)}?') else: return 'patchset applies.' diff --git a/orbit/radius.py b/orbit/radius.py index ab26f8f0..002fa2e5 100644 --- a/orbit/radius.py +++ b/orbit/radius.py @@ -448,6 +448,29 @@ def oops_button_hover(self): case OopsStatus.UNAVAILABLE: return "You have already used your oopsie" + def get_automated_feedback(self, attr): + if attr not in ['init', 'final'] or (gbl := getattr(self, attr)) is None: + return 'No submission' + + match attr: + case 'init': + due_date = int(self.assignment.initial_due_date) + case 'final': + due_date = int(self.assignment.final_due_date) + + if due_date < int(datetime.now().timestamp()): + return gbl.status + + match gbl.status[-1]: + case '.': + return 'Submission accepted' + case '?': + return 'Issues detected in patchset' + case '!': + return 'Submission rejected' + case _: + return '---' + def gradeable_row(self, item_name, gradeable, rightmost_col): return f""" @@ -480,8 +503,8 @@ def body(self): return f""" {self.gradeable_row('Final Submission', self.final, self.oopsie_button())} - Comments - - + Automated Feedback + {self.get_automated_feedback('final')} """ if (not self.init or @@ -491,14 +514,14 @@ def body(self): {self.gradeable_row('Initial Submission', self.init, self.oopsie_button())} Automated Feedback - - + {self.get_automated_feedback('init')} """ return f""" {self.gradeable_row('Initial Submission', self.init, self.oopsie_button())} Automated Feedback - - + {self.get_automated_feedback('init')} @@ -510,8 +533,8 @@ def body(self): {self.gradeable_row(self.peer2 + ' Peer Review', self.review2, '-') if self.peer2 else ''} {self.gradeable_row('Final Submission', self.final, '-')} - Comments - - + Automated Feedback + {self.get_automated_feedback('final')} """ From ddb25fe992f7bca63b201a87300e4fbe7b5b0304 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 16:08:49 -0400 Subject: [PATCH 02/26] mailman: factor out git.Repo.init() and git.Repo() into callers Signed-off-by: Joel Savitz --- mailman/patchset.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/mailman/patchset.py b/mailman/patchset.py index 8ffac792..b1ae71ec 100644 --- a/mailman/patchset.py +++ b/mailman/patchset.py @@ -19,9 +19,8 @@ def try_or_false(do, exc): return False -def tag_and_push(repo_path, tag_name): +def tag_and_push(repo, tag_name): try: - repo = git.Repo(repo_path) repo.create_tag(tag_name) repo.create_remote('grading', REMOTE_PUSH_URL) repo.git.push('grading', tags=True) @@ -37,8 +36,7 @@ def tag_and_push(repo_path, tag_name): *author_args, 'am', '--keep'] -def do_check(repo_path, cover_letter, patches): - repo = git.Repo.init(repo_path) +def do_check(repo, cover_letter, patches): whitespace_errors = [] def am_cover_letter(keep_empty=True): @@ -88,9 +86,9 @@ def do_git_am(extra_args=[]): def check(cover_letter, patches, submission_id): with tempfile.TemporaryDirectory() as repo_path: - status = do_check(repo_path, cover_letter, patches) + repo = git.Repo.init(repo_path) + status = do_check(repo, cover_letter, patches) if status[-1] == '!': - repo = git.Repo(repo_path) for patch in patches: patch_abspath = str(maildir / patch.msg_id) repo.git.execute(['git', *author_args, 'commit', '--allow-empty', '-F', patch_abspath]) @@ -111,7 +109,7 @@ def apply_peer_review(email, submission_id, review_id): '--single-branch', '--no-tags']) repo.git.execute([*args, patch_abspath]) - tag_and_push(repo_path, submission_id) + tag_and_push(repo, submission_id) except git.GitCommandError as e: print(e, file=sys.stderr) status = 'failed to apply peer review' From 5cf3bd9972f728afdf09f162b10c60d115a735b8 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 16:10:28 -0400 Subject: [PATCH 03/26] mailman: factor out author args into short config blocks Signed-off-by: Joel Savitz --- mailman/patchset.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/mailman/patchset.py b/mailman/patchset.py index b1ae71ec..6b737063 100644 --- a/mailman/patchset.py +++ b/mailman/patchset.py @@ -30,10 +30,8 @@ def tag_and_push(repo, tag_name): return False -author_args = ['-c', 'user.name=mailman', '-c', - 'user.email=mailman@mailman'] git_am_args = ['git', '-c', 'advice.mergeConflict=false', - *author_args, 'am', '--keep'] + 'am', '--keep'] def do_check(repo, cover_letter, patches): @@ -49,7 +47,7 @@ def am_cover_letter(keep_empty=True): git.GitCommandError): return "missing cover letter!" - repo.git.execute(["git", *author_args, "am", "--abort"]) + repo.git.execute(["git", "am", "--abort"]) if not try_or_false(lambda: am_cover_letter(keep_empty=True), git.GitCommandError): return ("missing cover letter and " @@ -67,7 +65,7 @@ def do_git_am(extra_args=[]): git.GitCommandError): continue - repo.git.execute(["git", *author_args, "am", "--abort"]) + repo.git.execute(["git", "am", "--abort"]) # Try again, if we succeed, count this patch as a whitespace error if try_or_false(lambda: do_git_am(), git.GitCommandError): @@ -87,12 +85,15 @@ def do_git_am(extra_args=[]): def check(cover_letter, patches, submission_id): with tempfile.TemporaryDirectory() as repo_path: repo = git.Repo.init(repo_path) + with repo.config_writer() as config: + config.set_value('user', 'name', 'mailman') + config.set_value('user', 'email', 'mailman@mailman') status = do_check(repo, cover_letter, patches) if status[-1] == '!': for patch in patches: patch_abspath = str(maildir / patch.msg_id) - repo.git.execute(['git', *author_args, 'commit', '--allow-empty', '-F', patch_abspath]) - tag_and_push(repo_path, submission_id) + repo.git.execute(['git', 'commit', '--allow-empty', '-F', patch_abspath]) + tag_and_push(repo, submission_id) return status @@ -108,6 +109,9 @@ def apply_peer_review(email, submission_id, review_id): multi_options=[f'--branch={review_id}', '--single-branch', '--no-tags']) + with repo.config_writer() as config: + config.set_value('user', 'name', 'mailman') + config.set_value('user', 'email', 'mailman@mailman') repo.git.execute([*args, patch_abspath]) tag_and_push(repo, submission_id) except git.GitCommandError as e: From 36a9cefc517cb67245bf81f96a4be05af911d4b8 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 14:21:33 -0400 Subject: [PATCH 04/26] mailman: add status strings to email ID tags Signed-off-by: Joel Savitz --- mailman/patchset.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mailman/patchset.py b/mailman/patchset.py index 6b737063..44a3854f 100644 --- a/mailman/patchset.py +++ b/mailman/patchset.py @@ -19,9 +19,9 @@ def try_or_false(do, exc): return False -def tag_and_push(repo, tag_name): +def tag_and_push(repo, tag_name, msg=None): try: - repo.create_tag(tag_name) + repo.create_tag(tag_name, message=msg) repo.create_remote('grading', REMOTE_PUSH_URL) repo.git.push('grading', tags=True) return True @@ -93,7 +93,7 @@ def check(cover_letter, patches, submission_id): for patch in patches: patch_abspath = str(maildir / patch.msg_id) repo.git.execute(['git', 'commit', '--allow-empty', '-F', patch_abspath]) - tag_and_push(repo, submission_id) + tag_and_push(repo, submission_id, msg=status) return status From e54a0bace51965defc770693e3be1c4f0cb322c3 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 13:06:54 -0400 Subject: [PATCH 05/26] orbit: read assignment grade from grading repo with git notes Store the grade for an assignment under the 'grade' ref linked to the final submission tag, e.g. to add a grade of '66' for user 'bob' assignment 'setup', one would run the following in the grading repo: $ git notes --ref=grade add setup_final_bob -m '66' $ git push origin refs/notes/* Assuming the connection to the grading repo is configured correctly, the grade will be immediately live for the student on the course dashboard Signed-off-by: Joel Savitz --- orbit/Containerfile | 1 + orbit/radius.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/orbit/Containerfile b/orbit/Containerfile index f083a0d7..9927359d 100644 --- a/orbit/Containerfile +++ b/orbit/Containerfile @@ -18,6 +18,7 @@ FROM alpine:3.20 AS orbit RUN apk update && apk upgrade && apk add \ py3-bcrypt \ + py3-gitpython \ py3-peewee \ py3-markdown \ uwsgi-python3 \ diff --git a/orbit/radius.py b/orbit/radius.py index 002fa2e5..6fc51fd1 100644 --- a/orbit/radius.py +++ b/orbit/radius.py @@ -4,6 +4,7 @@ import base64 import bcrypt +import git import html import markdown import os @@ -471,6 +472,17 @@ def get_automated_feedback(self, attr): case _: return '---' + def get_grade(self): + if self.final is None: + return '-' + tag = f'{self.final.assignment}_final_{self.final.user}' + repo = git.Repo('/var/lib/git/grading.git') + try: + grade = repo.git.execute(['git', 'notes', '--ref=grade', 'show', tag]) + return grade + except git.GitCommandError: + return '-' + def gradeable_row(self, item_name, gradeable, rightmost_col): return f""" @@ -531,7 +543,7 @@ def body(self): {self.gradeable_row(self.peer1 + ' Peer Review', self.review1, '-') if self.peer1 else ''} {self.gradeable_row(self.peer2 + ' Peer Review', self.review2, '-') if self.peer2 else ''} - {self.gradeable_row('Final Submission', self.final, '-')} + {self.gradeable_row('Final Submission', self.final, self.get_grade())} Automated Feedback {self.get_automated_feedback('final')} From 303ee893a0867a4671e6ec034934f391ede7795e Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 16:01:39 -0400 Subject: [PATCH 06/26] orbit: read peer review grade grade from grading repo with git notes Just like the previous commit adding this for final subs, but this time for peer reviews. Example workflow: $ git notes --ref=grade add setup_review1_bob -m '22' $ git notes --ref=grade add setup_review2_bob -m '66' $ git push origin refs/notes/* Fixes: #259 Signed-off-by: Joel Savitz --- orbit/radius.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/orbit/radius.py b/orbit/radius.py index 6fc51fd1..bfbcbc31 100644 --- a/orbit/radius.py +++ b/orbit/radius.py @@ -472,10 +472,10 @@ def get_automated_feedback(self, attr): case _: return '---' - def get_grade(self): - if self.final is None: + def get_grade(self, attr): + if attr not in ['final', 'review1', 'review2'] or (gbl := getattr(self, attr)) is None: return '-' - tag = f'{self.final.assignment}_final_{self.final.user}' + tag = f'{gbl.assignment}_{attr}_{gbl.user}' repo = git.Repo('/var/lib/git/grading.git') try: grade = repo.git.execute(['git', 'notes', '--ref=grade', 'show', tag]) @@ -541,9 +541,9 @@ def body(self): Submission ID Score - {self.gradeable_row(self.peer1 + ' Peer Review', self.review1, '-') if self.peer1 else ''} - {self.gradeable_row(self.peer2 + ' Peer Review', self.review2, '-') if self.peer2 else ''} - {self.gradeable_row('Final Submission', self.final, self.get_grade())} + {self.gradeable_row(self.peer1 + ' Peer Review', self.review1, self.get_grade('review1')) if self.peer1 else ''} + {self.gradeable_row(self.peer2 + ' Peer Review', self.review2, self.get_grade('review2')) if self.peer2 else ''} + {self.gradeable_row('Final Submission', self.final, self.get_grade('final'))} Automated Feedback {self.get_automated_feedback('final')} From d9c01d1f1fe669c0628be297b7037fe13159d2a0 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 16:25:30 -0400 Subject: [PATCH 07/26] orbit: cache the result of get_grade() in AsmtTable We are going to calculate the overall grade which will include multiple accesses to the grade values. By caching, we won't have to access the git repo more than once for the same information. Signed-off-by: Joel Savitz --- orbit/radius.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/orbit/radius.py b/orbit/radius.py index bfbcbc31..5d4cd61d 100644 --- a/orbit/radius.py +++ b/orbit/radius.py @@ -436,6 +436,9 @@ def __init__(self, assignment, oopsieness, peer1, peer2, init, self.review1 = review1 self.review2 = review2 self.final = final + self.review1_grade = None + self.review2_grade = None + self.final_grade = None def oops_button_hover(self): match self.oopsieness: @@ -475,10 +478,13 @@ def get_automated_feedback(self, attr): def get_grade(self, attr): if attr not in ['final', 'review1', 'review2'] or (gbl := getattr(self, attr)) is None: return '-' + if (cached_grade := getattr(self, f'{attr}_grade')) is not None: + return cached_grade tag = f'{gbl.assignment}_{attr}_{gbl.user}' repo = git.Repo('/var/lib/git/grading.git') try: grade = repo.git.execute(['git', 'notes', '--ref=grade', 'show', tag]) + setattr(self, f'{attr}_grade', grade) return grade except git.GitCommandError: return '-' From e31e2a9cb83c4018971a037d285d035c582a8624 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 16:35:22 -0400 Subject: [PATCH 08/26] orbit: calculate total score from final score and review scores If the oopsie was used for this assignment, just use the final score. Fixes #260 Signed-off-by: Joel Savitz --- orbit/radius.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/orbit/radius.py b/orbit/radius.py index 5d4cd61d..17b97621 100644 --- a/orbit/radius.py +++ b/orbit/radius.py @@ -489,6 +489,24 @@ def get_grade(self, attr): except git.GitCommandError: return '-' + def get_total_score(self): + oopsie_used_here = self.oopsieness == OopsStatus.USED_HERE + final_grade = self.get_grade('final') + + if oopsie_used_here: + return final_grade + + review1_grade = self.get_grade('review1') + review2_grade = self.get_grade('review2') + + if '-' in [final_grade, review1_grade, review2_grade]: + return '-' + + try: + return format(int(final_grade) * .8 + int(review1_grade) * .1 + int(review2_grade) * .1, '.1f') + except ValueError: + return "???" + def gradeable_row(self, item_name, gradeable, rightmost_col): return f""" @@ -561,7 +579,7 @@ def __str__(self): - + From 732b01d03ba76bacaa8264b00a9e292e3c6a9252 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 18:01:21 -0400 Subject: [PATCH 09/26] denis: add notes tied to initial and final submissions Right now this is just a fixed string. Soon, the note will contain automated test output. Signed-off-by: Joel Savitz --- denis/final.py | 4 +++- denis/initial.py | 4 +++- denis/utilities.py | 29 ++++++++++++++++++++++++++--- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/denis/final.py b/denis/final.py index cbee80df..3728d311 100755 --- a/denis/final.py +++ b/denis/final.py @@ -11,4 +11,6 @@ print(f'final subs for {assignment} released') -utilities.update_tags(assignment, 'final') +tags = utilities.update_tags(assignment, 'final') + +utilities.run_automated_checks(tags) diff --git a/denis/initial.py b/denis/initial.py index 5faa098c..71cf2d05 100755 --- a/denis/initial.py +++ b/denis/initial.py @@ -58,4 +58,6 @@ print(f'initial subs for {assignment} released') -utilities.update_tags(assignment, 'initial') +tags = utilities.update_tags(assignment, 'initial') + +utilities.run_automated_checks(tags) diff --git a/denis/utilities.py b/denis/utilities.py index 0120a7e0..09e34f74 100644 --- a/denis/utilities.py +++ b/denis/utilities.py @@ -38,6 +38,12 @@ def release_subs(sub_ids): input=journal_data, check=True) +def configure_repo(repo): + with repo.config_writer() as config: + config.set_value('user', 'name', 'denis') + config.set_value('user', 'email', 'denis@denis') + + def update_tags(assignment, component): grd_tbl = mailman.db.Gradeable subs = (grd_tbl.select() @@ -47,15 +53,15 @@ def update_tags(assignment, component): with tempfile.TemporaryDirectory() as repo_path: repo = git.Repo.clone_from(PULL_URL, repo_path) repo.create_remote(REMOTE_NAME, PUSH_URL) - repo.config_writer().set_value('user', 'name', 'denis').release() - (repo.config_writer().set_value('user', 'email', 'denis@denis') - .release()) + configure_repo(repo) if 'EMPTY' not in repo.tags: repo.git.commit('--allow-empty', '-m', 'No gradeable submission.') repo.create_tag('EMPTY') + updated_tags = [] for user in orbit.db.User.select(): new_tag_name = f'{assignment}_{component}_{user.username}' + updated_tags.append(new_tag_name) if new_tag_name in repo.tags: print('Potential issue? Attempted to create duplicate tag ' f'{new_tag_name}') @@ -70,3 +76,20 @@ def update_tags(assignment, component): repo.create_tag(new_tag_name, ref=to_promote.commit, message=msg) repo.git.push(REMOTE_NAME, tags=True) + + return updated_tags + + +def run_automated_checks(tags): + with tempfile.TemporaryDirectory() as repo_path: + repo = git.Repo.clone_from(PULL_URL, repo_path) + + remote = repo.create_remote(REMOTE_NAME, PUSH_URL) + remote.fetch('refs/notes/*:refs/notes/*') + configure_repo(repo) + + for tag in tags: + msg = 'Automated tests by denis' + repo.git.execute(['git', 'notes', '--ref=denis', 'add', tag, '-m', msg]) + + remote.push('refs/notes/*:refs/notes/*') From ff073de8a6f841543a18bdda49260efc854d4b7a Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 18:35:50 -0400 Subject: [PATCH 10/26] denis,orbit: jank implementation of automatic 0 for corrupt or missing patches. proof of concept Fixes #207 Signed-off-by: Joel Savitz --- denis/utilities.py | 30 ++++++++++++++++++++++++++++++ orbit/radius.py | 16 ++++++++++------ 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/denis/utilities.py b/denis/utilities.py index 09e34f74..53237131 100644 --- a/denis/utilities.py +++ b/denis/utilities.py @@ -80,6 +80,33 @@ def update_tags(assignment, component): return updated_tags +def check_corrupt_or_missing(repo, tag): + [assignment, component, user] = tag.split('_') + + grd_tbl = mailman.db.Gradeable + gradable = (grd_tbl.select() + .order_by(-grd_tbl.timestamp) + .where(grd_tbl.assignment == assignment) + .where(grd_tbl.component == component) + .where(grd_tbl.user == user)).first() + + msg = 'corruption and existence check' + msg += '\n' + msg += '------------------------------' + msg += '\n\n' + if not gradable or gradable.status[-1] == '!': + repo.git.execute(['git', 'notes', '--ref=grade', 'add', tag, '-m', '0']) + if not gradable: + msg += 'automatic 0 due to missing submission!' + else: + msg += 'automatic 0 due to corrupt submission!' + else: + msg += 'PATCHSET APPLIES' + + msg += '\n\n' + return msg + + def run_automated_checks(tags): with tempfile.TemporaryDirectory() as repo_path: repo = git.Repo.clone_from(PULL_URL, repo_path) @@ -90,6 +117,9 @@ def run_automated_checks(tags): for tag in tags: msg = 'Automated tests by denis' + msg += '\n\n' + msg += check_corrupt_or_missing(repo, tag) + # stop if corrupt (msg[-1] == '!') repo.git.execute(['git', 'notes', '--ref=denis', 'add', tag, '-m', msg]) remote.push('refs/notes/*:refs/notes/*') diff --git a/orbit/radius.py b/orbit/radius.py index 17b97621..ec990306 100644 --- a/orbit/radius.py +++ b/orbit/radius.py @@ -426,7 +426,7 @@ class OopsStatus: class AsmtTable: def __init__(self, assignment, oopsieness, peer1, peer2, init, - review1, review2, final): + review1, review2, final, user): self.assignment = assignment self.name = assignment.name self.oopsieness = oopsieness @@ -436,6 +436,7 @@ def __init__(self, assignment, oopsieness, peer1, peer2, init, self.review1 = review1 self.review2 = review2 self.final = final + self.user = user self.review1_grade = None self.review2_grade = None self.final_grade = None @@ -476,12 +477,15 @@ def get_automated_feedback(self, attr): return '---' def get_grade(self, attr): - if attr not in ['final', 'review1', 'review2'] or (gbl := getattr(self, attr)) is None: - return '-' + tag = f'{self.name}_{attr}_{self.user}' + repo = git.Repo('/var/lib/git/grading.git') + if attr not in ['final', 'review1', 'review2'] or getattr(self, attr) is None: + try: + return repo.git.execute(['git', 'notes', '--ref=grade', 'show', tag]) + except git.GitCommandError: + return '-' if (cached_grade := getattr(self, f'{attr}_grade')) is not None: return cached_grade - tag = f'{gbl.assignment}_{attr}_{gbl.user}' - repo = git.Repo('/var/lib/git/grading.git') try: grade = repo.git.execute(['git', 'notes', '--ref=grade', 'show', tag]) setattr(self, f'{attr}_grade', grade) @@ -653,7 +657,7 @@ def handle_dashboard(rocket): rev2 = asn_gradeables.where(grd_tbl.component == 'review2').first() final = asn_gradeables.where(grd_tbl.component == 'final').first() ret += str(AsmtTable(assignment, oopsieness, peer1, peer2, init, rev1, - rev2, final)) + rev2, final, rocket.session.username)) return rocket.respond(ret + '', 'Dashboard') From 3bb3f22b74570b7036a799563c150c3d284e11c1 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 23:16:25 -0400 Subject: [PATCH 11/26] denis: verify submitted and calculated diffstat Verify that the diffstat in a submitted cover letter matches the diffstat calculated with `git diff --stat --summary $(git rev-list --max-parents=0 )..` Fixes #111 Signed-off-by: Joel Savitz --- denis/utilities.py | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/denis/utilities.py b/denis/utilities.py index 53237131..5bf2b4f3 100644 --- a/denis/utilities.py +++ b/denis/utilities.py @@ -1,6 +1,7 @@ import git import subprocess import tempfile +import difflib import orbit.db import mailman.db @@ -107,6 +108,40 @@ def check_corrupt_or_missing(repo, tag): return msg +def check_diffstat(repo, tag): + msg = 'diffstat check' + msg += '\n' + msg += '--------------' + msg += '\n\n' + + root = repo.git.execute(['git', 'rev-list', '--max-parents=0', tag]) + calculated_diffstat = repo.git.execute(['git', 'diff', '--stat', '--summary', f'{root}..{tag}']) + cover = repo.git.execute(['git', 'show', root]) + cover_lines = [line.strip() for line in cover.split('\n')] + + rev_diffstat = [] + last = None + collect = False + for i in reversed(cover_lines): + if collect: + if len(i) == 0: + break + rev_diffstat.append(f' {i}') + if len(i) == 0 and last == '--': + collect = True + last = i + + cover_diffstat = '\n'.join(reversed(rev_diffstat)) + diff_diffstat = '\n'.join(difflib.unified_diff(calculated_diffstat.split(), cover_diffstat.split())) + + msg += 'diffstat diff' + msg += '\n' + msg += diff_diffstat if len(diff_diffstat) > 0 else 'NO DIFFERENCE: DIFFSTAT VERIFIED' + msg += '\n\n' + + return msg + + def run_automated_checks(tags): with tempfile.TemporaryDirectory() as repo_path: repo = git.Repo.clone_from(PULL_URL, repo_path) @@ -119,7 +154,11 @@ def run_automated_checks(tags): msg = 'Automated tests by denis' msg += '\n\n' msg += check_corrupt_or_missing(repo, tag) - # stop if corrupt (msg[-1] == '!') + + if msg[-3] != '!': + msg += '\n\n' + msg += check_diffstat(repo, tag) + repo.git.execute(['git', 'notes', '--ref=denis', 'add', tag, '-m', msg]) remote.push('refs/notes/*:refs/notes/*') From 1ef626beb76a3bd5d8d3ca7fe7c7b29095b8b74d Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 23 May 2025 00:30:13 -0400 Subject: [PATCH 12/26] denis: verify DCO in each patch in patchset compare DCO in each patch and cover letter to one created by combining the known username and hostname. Fixes #110 Signed-off-by: Joel Savitz --- container-compose.yml | 1 + denis/utilities.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/container-compose.yml b/container-compose.yml index bbc50556..8b92f26e 100644 --- a/container-compose.yml +++ b/container-compose.yml @@ -185,6 +185,7 @@ services: - orbit_source=./orbit environment: TZ: ${SINGULARITY_TIMEZONE} + HOSTNAME: ${SINGULARITY_HOSTNAME} volumes: - type: volume source: denis-db diff --git a/denis/utilities.py b/denis/utilities.py index 5bf2b4f3..4790e953 100644 --- a/denis/utilities.py +++ b/denis/utilities.py @@ -1,3 +1,5 @@ +import re +import os import git import subprocess import tempfile @@ -108,6 +110,45 @@ def check_corrupt_or_missing(repo, tag): return msg +def check_signed_off_by(repo, tag): + [_, _, user] = tag.split('_') + hostname = os.getenv("HOSTNAME") + + msg = 'signed off by check' + msg += '\n' + msg += '-------------------' + msg += '\n\n' + + commits = reversed(repo.git.execute(['git', 'rev-list', tag]).split('\n')) + + expected_dco = f'Signed-off-by: {user} <{user}@{hostname}>' + + missing = [] + malformed = [] + for i, commit in enumerate(commits): + patch = repo.git.execute(['git', 'show', commit]) + + match = re.search(r'^\s+(Signed-off-by:\s+.+\s+<.+>)$', patch, re.MULTILINE) + if match: + found_dco = match.group(1) + if expected_dco == found_dco: + continue + malformed.append(f'malformed line {found_dco} in patch {i}\n') + else: + missing.append(f'{i}') + + if (n := len(missing)) > 0: + msg += f'Signed-off-by: missing from patch{"es" if n > 1 else ""} {",".join(missing)}\n' + for mal in malformed: + msg += mal + + if len(missing) == len(malformed) == 0: + msg += 'ALL PATCHES SIGNED OFF CORRECTLY\n' + + msg += '\n' + return msg + + def check_diffstat(repo, tag): msg = 'diffstat check' msg += '\n' @@ -158,6 +199,7 @@ def run_automated_checks(tags): if msg[-3] != '!': msg += '\n\n' msg += check_diffstat(repo, tag) + msg += check_signed_off_by(repo, tag) repo.git.execute(['git', 'notes', '--ref=denis', 'add', tag, '-m', msg]) From 45a7aba341a079be0c7564ab9b201295d1dbb0d5 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 12:40:51 -0400 Subject: [PATCH 13/26] denis: check subject tag format Check RFC/lack thereof, PATH, vN consistency, and correct indexes Fixes: #262 Signed-off-by: Joel Savitz --- denis/utilities.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/denis/utilities.py b/denis/utilities.py index 4790e953..6529c2f0 100644 --- a/denis/utilities.py +++ b/denis/utilities.py @@ -1,6 +1,7 @@ import re import os import git +import copy import subprocess import tempfile import difflib @@ -183,6 +184,71 @@ def check_diffstat(repo, tag): return msg +def check_subject_tag(repo, tag): + [_, component, _] = tag.split('_') + + msg = 'subject tag check' + msg += '\n' + msg += '-----------------' + msg += '\n\n' + + commits = reversed(repo.git.execute(['git', 'rev-list', tag]).split('\n')) + # from 0/n .. n/n + expected_max_index = str(len(list(copy.copy(commits))) - 1) + + rfc_offset = 0 + found_version = None + bad = {} + for i, commit in enumerate(commits): + patch = repo.git.execute(['git', 'show', commit]) + + match = re.search(r'^\s*\[(.*)\].*', patch, re.MULTILINE) + if not match: + msg += f'patch {i} no tag found!\n' + bad[i] = True + continue + subj_tag = match.group(1) + msg += f'patch {i} tag {subj_tag}\n' + subj_tag_parts = subj_tag.split(' ') + match component: + case 'initial' if subj_tag_parts[0] == 'RFC': + msg += 'initial submission correctly labeled RFC\n' + rfc_offset = 1 + case 'initial' if subj_tag_parts[0] == 'PATCH': + msg += 'initial submission missing RFC!\n' + bad[i] = True + case 'final' if subj_tag_parts[0] == 'RFC': + msg += 'final submission incorrectly labeled RFC!\n' + rfc_offset = 1 + bad[i] = True + case 'final' if subj_tag_parts[0] == 'PATCH': + msg += 'final submission correctly non RFC\n' + + if subj_tag_parts[rfc_offset] != 'PATCH': + msg += 'tag missing "PATCH" in expected place\n' + bad[i] = True + + if found_version is None: + found_version = subj_tag_parts[rfc_offset+1] + elif (this_version := subj_tag_parts[rfc_offset+1]) != found_version: + msg += f'tag version mismatch: expected {found_version} found {this_version}!\n' + bad[i] = True + + [this_index, max_index] = subj_tag_parts[rfc_offset+2].split('/') + if this_index != str(i): + msg += f'patch index mismatch: expected {i} found {this_index}' + bad[i] = True + + if max_index != expected_max_index: + msg += f'patch max index mismatch: expected {expected_max_index} found {max_index}' + bad[i] = True + + if not any(bad): + msg += 'ALL SUBJECT TAGS IN CORRECT FORMAT\n' + + return msg + + def run_automated_checks(tags): with tempfile.TemporaryDirectory() as repo_path: repo = git.Repo.clone_from(PULL_URL, repo_path) @@ -200,6 +266,7 @@ def run_automated_checks(tags): msg += '\n\n' msg += check_diffstat(repo, tag) msg += check_signed_off_by(repo, tag) + msg += check_subject_tag(repo, tag) repo.git.execute(['git', 'notes', '--ref=denis', 'add', tag, '-m', msg]) From 962c4d927508646e31fae30264978f746ae7be4f Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 18:05:16 -0400 Subject: [PATCH 14/26] orbit: dashboard: implement human feedback section Grader can write to this section via the feedback ref on the relevant final submission tag, e.g.: $ git notes --ref=feedback add setup_final_bob -m 'Feedback message' Signed-off-by: Joel Savitz --- orbit/radius.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/orbit/radius.py b/orbit/radius.py index ec990306..d1b7e798 100644 --- a/orbit/radius.py +++ b/orbit/radius.py @@ -476,6 +476,14 @@ def get_automated_feedback(self, attr): case _: return '---' + def get_human_feedback(self): + tag = f'{self.name}_final_{self.user}' + repo = git.Repo('/var/lib/git/grading.git') + try: + return repo.git.execute(['git', 'notes', '--ref=feedback', 'show', tag]) + except git.GitCommandError: + return '-' + def get_grade(self, attr): tag = f'{self.name}_{attr}_{self.user}' repo = git.Repo('/var/lib/git/grading.git') @@ -546,6 +554,10 @@ def body(self): + + + + """ if (not self.init or (int(datetime.now().timestamp()) @@ -576,6 +588,10 @@ def body(self): + + + + """ def __str__(self): From ac1e7711d9c5934d4f9a1a2e7a90bf29c7e09e03 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 19:18:19 -0400 Subject: [PATCH 15/26] denis/configure: inform user when dumping dirty When the assignment database is altered, the due date waiters must be restarted in order for the new due dates to be operative using: $ denis/configure.sh reload Currently there is no way to know that one has forgotten to run this command. Add a warning message to the output of: $ denis/configure.sh dump to indicate when one has altered the database but not yet operationalized their intentions. Fixes #257 Signed-off-by: Joel Savitz --- denis/configure.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/denis/configure.py b/denis/configure.py index c18060d9..9fc41037 100755 --- a/denis/configure.py +++ b/denis/configure.py @@ -2,6 +2,8 @@ from datetime import datetime from argparse import ArgumentParser as ap +from pathlib import Path +import os import db @@ -67,12 +69,17 @@ def add_final(parser, required=True): subparser_func(**kwargs) +def dirty(): + Path('/tmp/dirty').touch() + + def create(assignment, initial, peer_review, final): try: db.Assignment.create(name=assignment, initial_due_date=initial, peer_review_due_date=peer_review, final_due_date=final) + dirty() except db.peewee.IntegrityError: print('cannot create assignment with duplicate name') @@ -92,6 +99,8 @@ def alter(assignment, initial, peer_review, final): .where(db.Assignment.name == assignment)) if query.execute() < 1: print(f'no such assignment {assignment}') + else: + dirty() def remove(assignment): @@ -100,6 +109,8 @@ def remove(assignment): .where(db.Assignment.name == assignment)) if query.execute() < 1: print(f'no such assignment {assignment}') + else: + dirty() def dump(fmt_iso): @@ -107,6 +118,9 @@ def timestamp_to_formatted(timestamp): dt = datetime.fromtimestamp(timestamp).astimezone() return dt.isoformat() if fmt_iso else dt.strftime('%a %b %d %Y %T %Z (%z)') + if os.path.exists('/tmp/dirty'): + print('WARNING: Denis database is dirty, reload to update waiters') + print(' --- Assignments ---') for asn in db.Assignment.select(): print(f'''{asn.name}: @@ -119,6 +133,8 @@ def reload(): import os import signal os.kill(1, signal.SIGUSR1) + if os.path.exists('/tmp/dirty'): + os.remove('/tmp/dirty') if __name__ == '__main__': From fb5311e8974bb4e2ea1574bfdf95472d399d017e Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 21:12:55 -0400 Subject: [PATCH 16/26] mailman: reject patchsets modifying files outside student directory Any patchset containing a hunk modifying a file outside of a directory named after their username at the top level of the submissions repository, as specified by the DCO in that particular patch, is rejected immediately upon submission. If no additional patchset is submitted before a deadline, the student will automatically receive a zero. Additionally, patches lacking a DCO at submission time are rejected. Fixes #145 Signed-off-by: Joel Savitz --- mailman/patchset.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/mailman/patchset.py b/mailman/patchset.py index 44a3854f..5ca168be 100644 --- a/mailman/patchset.py +++ b/mailman/patchset.py @@ -1,3 +1,4 @@ +import re import git import pathlib import sys @@ -56,6 +57,24 @@ def am_cover_letter(keep_empty=True): for i, patch in enumerate(patches): patch_abspath = str(maildir / patch.msg_id) + with open(patch_abspath, 'r') as patch_file: + patch_content = patch_file.read() + match = re.search(r'^\s+Signed-off-by:\s+.+\s+<(.+)@.+>$', patch_content, re.MULTILINE) + if match: + found_author = match.group(1) + else: + return f'illegal patch {i+1}: missing Signed-off-by line!' + + changelines = list(filter(lambda line: line.startswith('--- ') or line.startswith('+++ '), patch_content.split('\n'))) + for change in changelines: + file = change.split(' ')[1].strip() + if file == '/dev/null': + continue + first_dir = file.split('/')[1] + if first_dir != found_author: + file_fixed = file[2:] + return f'illegal patch {i+1}: permission denied for path {file_fixed}!' + # Try and apply and fail if there are whitespace errors def do_git_am(extra_args=[]): repo.git.execute([*git_am_args, *extra_args, patch_abspath]), From 8edab5acf6197b941a8090b5dc1a5b734b3b64e3 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 21:31:32 -0400 Subject: [PATCH 17/26] denis: automatic zero for peer review when no submission is made Run the peer review tags from denis/peer_review.py through the same run_automated_checks() as the initial and final tags, but skip the checks that don't make sense. This opens up the door for checking peer review specific requirements like the Acked-by and Nacked-by. Signed-off-by: Joel Savitz --- denis/peer_review.py | 6 ++++-- denis/utilities.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/denis/peer_review.py b/denis/peer_review.py index 72bc418a..32095a27 100755 --- a/denis/peer_review.py +++ b/denis/peer_review.py @@ -14,5 +14,7 @@ print(f'peer review subs for {assignment} released') -utilities.update_tags(assignment, 'review1') -utilities.update_tags(assignment, 'review2') +tags1 = utilities.update_tags(assignment, 'review1') +tags2 = utilities.update_tags(assignment, 'review2') + +utilities.run_automated_checks(tags1 + tags2, peer=True) diff --git a/denis/utilities.py b/denis/utilities.py index 6529c2f0..529ee753 100644 --- a/denis/utilities.py +++ b/denis/utilities.py @@ -249,7 +249,7 @@ def check_subject_tag(repo, tag): return msg -def run_automated_checks(tags): +def run_automated_checks(tags, peer=False): with tempfile.TemporaryDirectory() as repo_path: repo = git.Repo.clone_from(PULL_URL, repo_path) @@ -262,7 +262,7 @@ def run_automated_checks(tags): msg += '\n\n' msg += check_corrupt_or_missing(repo, tag) - if msg[-3] != '!': + if msg[-3] != '!' and not peer: msg += '\n\n' msg += check_diffstat(repo, tag) msg += check_signed_off_by(repo, tag) From 2e26bec4a46ae0f53338dfe9a854168183e0f0e5 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Sat, 31 May 2025 13:49:58 -0400 Subject: [PATCH 18/26] mailman,denis: add simple patchset rubric based rejection support Add support for checking whether changes made in each patch of a patchset conform to the expected set of changes. Only simple changes are supported, i.e. either required file creation or required file modification, both with the path to the file precisely specified. Mailman also requires the patchset to contain the expected number of patches To create a rubric, make commits on some repo, perhaps submissions, that make the expected changes for the expected number of patches N and run: $ $SINGULARITY_DIR/create_rubric.py -n $N > rubric To generate a file that can be passed to denis via: $ denis/configure.sh -r /path/to/rubric <...other args...> The rubric for an assignment can be viewed with denis/configure.sh dump. The rubric is enforced at submissison time, i.e. mailman rejects nonconforming patchsets immediately, marks them as corrupt, and creates a submission ID tag that contains the patch emails as empty patches. If no follow up submission is made, the student will receive a 0 for that component of the assignment. Fixes #109 Signed-off-by: Joel Savitz --- create_rubric.py | 24 ++++++++++++++++++++++++ denis/configure.py | 31 +++++++++++++++++++++++++++---- denis/configure.sh | 24 ++++++++++++++++++++++++ denis/db.py | 1 + mailman/patchset.py | 42 +++++++++++++++++++++++++++++++++++++++--- mailman/submit.py | 2 +- 6 files changed, 116 insertions(+), 8 deletions(-) create mode 100755 create_rubric.py diff --git a/create_rubric.py b/create_rubric.py new file mode 100755 index 00000000..c218b17c --- /dev/null +++ b/create_rubric.py @@ -0,0 +1,24 @@ +#!/bin/env python + +from argparse import ArgumentParser as ap +import git +import os + + +def main(): + parser = ap(prog='create_rubric', description='create rubric') + parser.add_argument('-n', type=int, help='number of patches to examine', + required=True) + + args = parser.parse_args() + + repo = git.Repo(os.getcwd()) + for i in range(args.n): + patch = repo.git.execute(['git', 'show', f'HEAD~{args.n-i-1}']) + changelines = list(filter(lambda line: line.startswith('--- ') or line.startswith('+++ '), patch.split('\n'))) + changeline_pairs = {(fromfile, tofile): 0 for fromfile, tofile in zip(changelines[::2], changelines[1::2])} + print(f'{"[" if i == 0 else ""}{changeline_pairs}{"," if i < args.n - 1 else "]"}') + + +if __name__ == '__main__': + main() diff --git a/denis/configure.py b/denis/configure.py index 9fc41037..bc3059f7 100755 --- a/denis/configure.py +++ b/denis/configure.py @@ -34,6 +34,11 @@ def add_final(parser, required=True): help='Final submission due date timetamp', required=required) + def add_rubric(parser): + parser.add_argument('-r', '--rubric', + type=str, + help='Rubric to enforce on incoming patchsets') + command_parsers = parser.add_subparsers(dest='command', required=True) create_parser = command_parsers.add_parser('create') @@ -41,12 +46,14 @@ def add_final(parser, required=True): add_initial(create_parser) add_peer_review(create_parser) add_final(create_parser) + add_rubric(create_parser) alter_parser = command_parsers.add_parser('alter') add_assignment(alter_parser) add_initial(alter_parser, required=False) add_peer_review(alter_parser, required=False) add_final(alter_parser, required=False) + add_rubric(alter_parser) remove_parser = command_parsers.add_parser('remove') add_assignment(remove_parser) @@ -73,18 +80,31 @@ def dirty(): Path('/tmp/dirty').touch() -def create(assignment, initial, peer_review, final): +# since the rubric is copied into the container by the wrapper script, +# we always place it in /tmp/rubric to simplify things +def load_rubric(): + try: + with open('/tmp/rubric', 'r') as rubric_file: + os.unlink('/tmp/rubric') + return rubric_file.read() + except FileNotFoundError: + return None + + +def create(assignment, initial, peer_review, final, rubric): + rubric = load_rubric() try: db.Assignment.create(name=assignment, initial_due_date=initial, peer_review_due_date=peer_review, - final_due_date=final) + final_due_date=final, + rubric=rubric) dirty() except db.peewee.IntegrityError: print('cannot create assignment with duplicate name') -def alter(assignment, initial, peer_review, final): +def alter(assignment, initial, peer_review, final, rubric): alterations = {} if initial is not None: alterations[db.Assignment.initial_due_date] = initial @@ -92,6 +112,8 @@ def alter(assignment, initial, peer_review, final): alterations[db.Assignment.peer_review_due_date] = peer_review if final is not None: alterations[db.Assignment.final_due_date] = final + if (rubric := load_rubric()) is not None: + alterations[db.Assignment.rubric] = rubric if not alterations: return print('At least one new date must be specified') query = (db.Assignment @@ -126,7 +148,8 @@ def timestamp_to_formatted(timestamp): print(f'''{asn.name}: \tInitial:\t{timestamp_to_formatted(asn.initial_due_date)} \tPeer Review:\t{timestamp_to_formatted(asn.peer_review_due_date)} -\tFinal:\t\t{timestamp_to_formatted(asn.final_due_date)}''') +\tFinal:\t\t{timestamp_to_formatted(asn.final_due_date)} +{"\tRubric:\n" + str(asn.rubric) if asn.rubric else ""}''') def reload(): diff --git a/denis/configure.sh b/denis/configure.sh index f22f4950..b629a08d 100755 --- a/denis/configure.sh +++ b/denis/configure.sh @@ -1,7 +1,31 @@ #!/bin/sh set -e +PODMAN=${PODMAN:-podman} COMPOSE=${COMPOSE:-podman-compose} +# a rubric is passed as a path to a file, +# but the main script is executed inside the container +# so copy any rubric into the container, +# if one is specified. +get_next= +rubric_file= +for arg in "$@" +do + if [ "$arg" = '-r' ] + then + get_next=yes + elif [ -n "$get_next" ] + then + rubric_file=$arg + fi +done + +if [ -n "$rubric_file" ] +then + ${PODMAN} cp "$rubric_file" singularity_denis_1:/tmp/rubric +fi + + ${COMPOSE} exec denis ./configure.py "$@" diff --git a/denis/db.py b/denis/db.py index 7dcf00ac..131a6cdd 100755 --- a/denis/db.py +++ b/denis/db.py @@ -15,6 +15,7 @@ class Assignment(BaseModel): initial_due_date = peewee.IntegerField() peer_review_due_date = peewee.IntegerField() final_due_date = peewee.IntegerField() + rubric = peewee.TextField(null=True) class PeerReviewAssignment(BaseModel): diff --git a/mailman/patchset.py b/mailman/patchset.py index 5ca168be..92a9be04 100644 --- a/mailman/patchset.py +++ b/mailman/patchset.py @@ -2,6 +2,7 @@ import git import pathlib import sys +import ast import tempfile REMOTE_PUSH_URL = 'http://git:8000/cgi-bin/git-receive-pack/grading.git' @@ -35,7 +36,7 @@ def tag_and_push(repo, tag_name, msg=None): 'am', '--keep'] -def do_check(repo, cover_letter, patches): +def do_check(repo, cover_letter, patches, asn): whitespace_errors = [] def am_cover_letter(keep_empty=True): @@ -54,6 +55,17 @@ def am_cover_letter(keep_empty=True): return ("missing cover letter and " "first patch failed to apply!") + rubric = None + if rubric_text := asn.rubric: + try: + rubric = ast.literal_eval(rubric_text) + except SyntaxError: + pass + + # check for correct # of patches in patchset + if rubric and (rubric_count := len(rubric)) != (sub_count := len(patches)): + return f'patch count {sub_count} violates expected rubric patch count of {rubric_count}!' + for i, patch in enumerate(patches): patch_abspath = str(maildir / patch.msg_id) @@ -75,6 +87,30 @@ def am_cover_letter(keep_empty=True): file_fixed = file[2:] return f'illegal patch {i+1}: permission denied for path {file_fixed}!' + # we assume first level directory is correct by this point + # so we can replace it with some random template value + # requirments: file is either being created of modified + # therefore we assume some second of a pair contains the template author + # rubric is a list of dictionaries mapping changepair tuples + # to an integer counting the number of times that changepair is found + if rubric: + other = list(rubric[0].keys())[0][1].split('/')[1] + for j in range(int(len(changelines)/2)): + changepair = [changelines[2*j], changelines[2*j+1]] + if changepair[0] != '--- /dev/null/': + changepair[0] = changepair[0].replace(found_author, other) + if changepair[1] != '+++ /dev/null/': + changepair[1] = changepair[1].replace(found_author, other) + try: + rubric[i][tuple(changepair)] += 1 + except KeyError: + pass + # a changepair (e.g. ('--- fromfile', '+++ tofile') may appear + # more than in the rubric o if modifications to a file + # are more sparse than in the example used to generate the rubric + if any(count < 1 for count in rubric[i].values()): + return f'patch {i+1} violates the assignment rubric!' + # Try and apply and fail if there are whitespace errors def do_git_am(extra_args=[]): repo.git.execute([*git_am_args, *extra_args, patch_abspath]), @@ -101,13 +137,13 @@ def do_git_am(extra_args=[]): return 'patchset applies.' -def check(cover_letter, patches, submission_id): +def check(cover_letter, patches, submission_id, asn): with tempfile.TemporaryDirectory() as repo_path: repo = git.Repo.init(repo_path) with repo.config_writer() as config: config.set_value('user', 'name', 'mailman') config.set_value('user', 'email', 'mailman@mailman') - status = do_check(repo, cover_letter, patches) + status = do_check(repo, cover_letter, patches, asn) if status[-1] == '!': for patch in patches: patch_abspath = str(maildir / patch.msg_id) diff --git a/mailman/submit.py b/mailman/submit.py index c7f69f3b..41555bcf 100755 --- a/mailman/submit.py +++ b/mailman/submit.py @@ -65,7 +65,7 @@ def set_status(status): gr_db = db.Gradeable if asn := asn_db.get_or_none(asn_db.name == emails[0].rcpt): cover_letter, *patches = emails - status = patchset.check(cover_letter, patches, logfile) + status = patchset.check(cover_letter, patches, logfile, asn) if len(emails) < 2: return set_status('missing patches') typ = ('initial' if timestamp < asn.initial_due_date From a3cdb8beb8c9574217e1daa4184a1624a5311ae2 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Thu, 22 May 2025 10:40:40 -0400 Subject: [PATCH 19/26] test-sub.sh: a new testing script Setup the submissions repo and go through the entire patchset submission process, exercising all return paths of mailman/patchset.py Includes intitial, peer review, and final submissions Signed-off-by: Joel Savitz --- script-lint.sh | 1 + test-sub.sh | 503 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 504 insertions(+) create mode 100755 test-sub.sh diff --git a/script-lint.sh b/script-lint.sh index 10cf94fa..455ca2a9 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -7,6 +7,7 @@ set -ex shellcheck script-lint.sh shellcheck test.sh +shellcheck test-sub.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/test-sub.sh b/test-sub.sh new file mode 100755 index 00000000..6ddd7eaa --- /dev/null +++ b/test-sub.sh @@ -0,0 +1,503 @@ +#!/bin/bash + +set -exuo pipefail + +SCRIPT_DIR=$(dirname "$0") + +cd "$SCRIPT_DIR" + +get_git_port() { podman port singularity_git_1 | awk -F':' '{ print $2 }' ; } ; + +# nuke grading repo inside container +# shellcheck disable=SC2016 +podman-compose exec git ash -c 'cd /var/lib/git/grading.git && for t in $(git tag); do git tag -d $t; done' + +# setup submissions repo +rm -rf repos/submissions +git/admin.sh submissions "course submissions repository" +pushd repos +git init --bare submissions +echo "course submissions repository" > submissions/description +git init submissions_init +pushd submissions_init +echo "# submissions" > README.md +git add README.md +git -c user.name=singularity -c user.email=singularity@singularity commit -m 'init submissions repo' +git push ../submissions master +popd +rm -rf submissions_init +pushd submissions +git push --mirror http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/submissions +popd +popd + +# create bob,bib,bab users or change their passwords to builder +orbit/warpdrive.sh -u bob -p builder -n || orbit/warpdrive.sh -u bob -p builder -m +orbit/warpdrive.sh -u bib -p builder -n || orbit/warpdrive.sh -u bob -p builder -m +orbit/warpdrive.sh -u bab -p builder -n || orbit/warpdrive.sh -u bob -p builder -m + +WORKDIR=$(mktemp -d) + +cat << EOF > "$WORKDIR"/setup_rubric +[{('--- /dev/null', '+++ b/bob/setup/work'): 0}, +{('--- a/bob/setup/work', '+++ b/bob/setup/work'): 0}] +EOF + +# create or recreate setup assignment +if denis/configure.sh dump | grep -q "^setup:"; then + denis/configure.sh remove -a setup +fi +denis/configure.sh create -a setup -i "$(date -d "15 secs" +%s)" -p "$(date -d "25 secs" +%s)" -f "$(date -d "30 secs" +%s)" -r "$WORKDIR"/setup_rubric +denis/configure.sh reload + +# setup assignment environment +pushd "$WORKDIR" + +mkdir certs +pushd certs +podman volume export singularity_ssl-certs > certs.tar +tar xf certs.tar +popd +git clone http://localhost:"$(get_git_port)"/submissions + +# bob env setup + +pushd submissions +git config user.name bob +git config user.email bob@localhost.localdomain +git config sendemail.smtpUser bob +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# setup_intial_bob patchsets + +# good patchset +pushd submissions +git checkout --orphan setup_initial_bob_good +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Good patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/Good patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# corrupt patchset +pushd submissions +git checkout --orphan setup_initial_bob_corrupt +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +echo "ghi" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add ghi to work' + +git format-patch --rfc --cover-letter -v1 -3 +rm v1-0002-*.patch + +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Corrupt patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/Corrupt patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# patchset with whitespace errors +pushd submissions +git checkout --orphan setup_initial_bob_whitespace +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def " >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Patchset with whitespace errors/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/Patchset with whitespace errors/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# patchset with no cover letter +pushd submissions +git checkout --orphan setup_initial_bob_nocover +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc -v1 -2 + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# patchset with no cover letter and corrupt first patch +pushd submissions +git checkout --orphan setup_initial_bob_nocover-corrupt +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +echo "ghi" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add ghi to work' + +git format-patch --rfc -v1 -2 + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd + +# end of setup_initial_bob patchsets + +# bab env setup + +pushd submissions +git config user.name bab +git config user.email bab@localhost.localdomain +git config sendemail.smtpUser bab +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# setup_initial_bab submissions + +# bab good patchset +pushd submissions +git checkout --orphan setup_initial_bab_good +git rm -rf ./* +mkdir -p bab/setup + +echo "abc" > bab/setup/work +git add bab/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bab/setup/work +git add bab/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bab: Good patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/bab: Good patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm -rf ./*.patch +popd + +# end of bab initial submission + +# bib env setup +pushd submissions +git config user.name bib +git config user.email bib@localhost.localdomain +git config sendemail.smtpUser bib +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# bib initial submissions + +# bib good patchset +pushd submissions +git checkout --orphan setup_initial_bib_good +git rm -rf ./* +mkdir -p bib/setup + +echo "abc" > bib/setup/work +git add bib/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bib/setup/work +git add bib/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bib: Good patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/bib: Good patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# end of bib initial submission + +# investigate grading repo +git clone http://localhost:"$(get_git_port)"/grading.git +pushd grading +git fetch --tag +git tag + + +STATUSES=( +'patchset applies.' +'patch 2 failed to apply!' +'whitespace error patch 2?' +'missing cover letter!' +'missing cover letter and first patch failed to apply!' +'patchset applies.' +'patchset applies.') + +# submitted patchses should have statuses in this order +# assumption: first ID tag is first patch submitted by this script and IDs increase monotonically +i=0 +for t in $(git tag); do + git show -s --oneline "$t" | grep -q "${STATUSES[$i]}" + if [[ $i == 5 ]]; then + PEER1_ID=$t + elif [[ $i == 6 ]]; then + PEER2_ID=$t + fi + i=$((i+1)) +done + +popd + +echo "wait for initial submission deadline (10 secs)" +sleep 10 + +# bob env setup + +pushd submissions +git config user.name bob +git config user.email bob@localhost.localdomain +git config sendemail.smtpUser bob +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + + +# setup bob peer review +pushd submissions +git checkout --orphan setup_reviews_bob +git rm -rf ./* +# mkdir -p bob/setup (must be able to omit if missing) +echo ORPHAN > ORPHAN +git add ORPHAN +git commit -m 'orphan base' + +# submit peer review 1 + +cat < review1 +Subject: setup review 1 for bab +In-Reply-To: <${PEER1_ID::-1}1@localhost.localdomain> + +Looks good to me + +Acked-by: bob +PEER_REVIEW1 + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bab@localhost.localdomain review1 +rm review1 + +# end peer review 1 + +# submit peer review 2 + +cat < review2 +Subject: setup review 2 for bib +In-Reply-To: <${PEER2_ID::-1}2@localhost.localdomain> + +Looks good to me + +Acked-by: bob +PEER_REVIEW2 + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bib@localhost.localdomain review2 +rm review2 + +# end peer review 2 + +# end bob peer review +popd +sleep 1 + +# bob env setup + +pushd submissions +git config user.name bob +git config user.email bob@localhost.localdomain +git config sendemail.smtpUser bob +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# bob make final submission + +# good final patchset +pushd submissions +git checkout --orphan setup_final_bob_good +git rm -rf ./* +mkdir -p bob/setup + +echo "abc" > bob/setup/work +git add bob/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bob/setup/work +git add bob/setup/work +git commit -sm 'add def to work' + +git format-patch --cover-letter -v2 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Good patchset final/g' v2-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/Good patchset final/g' v2-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# end bob final submission + +# bib env setup + +pushd submissions +git config user.name bib +git config user.email bib@localhost.localdomain +git config sendemail.smtpUser bib +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + +# bib final submission labeled RFC +pushd submissions +git checkout --orphan setup_final_bib_rfc +git rm -rf ./* +mkdir -p bib/setup + +echo "abc" > bib/setup/work +git add bib/setup/work +git commit -sm 'add abc to work' + +echo "def" >> bib/setup/work +git add bib/setup/work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bib: RFC patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/bib: RFC patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# end of bib final submission labeled RFC + + +# bab env setup + +pushd submissions +git config user.name bab +git config user.email bab@localhost.localdomain +git config sendemail.smtpUser bab +git config sendemail.smtpPass builder +git config sendemail.smtpserver localhost.localdomain +git config sendemail.smtpserverport 465 +git config sendemail.smtpencryption ssl +popd + + +# bab final submission in wrong directory +pushd submissions +git checkout --orphan setup_final_bab_illegal +git rm -rf ./* +mkdir -p bib/setup + +echo "abc" > work +git add work +git commit -sm 'add abc to work' + +echo "def" >> work +git add work +git commit -sm 'add def to work' + +git format-patch --rfc --cover-letter -v1 -2 +sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bab: Illegal patchset/g' v1-0000-cover-letter.patch +sed -i 's/\*\*\* BLURB HERE \*\*\*/bab: Illegal patchset/g' v1-0000-cover-letter.patch + +git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch +rm ./*.patch +popd +sleep 1 + +# end of bab final submission in wrong directory + + +echo "wait for final submission deadline (10 secs)" +sleep 10 + +pushd grading +git fetch --tag +sleep 2 +git fetch -f origin refs/notes/*:refs/notes/* +git remote set-url --push origin http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/grading.git +git notes --ref=grade add setup_final_bob -m '66' +git notes --ref=grade add setup_review1_bob -m '22' +git notes --ref=grade add setup_review2_bob -m '99' +git notes --ref=feedback add setup_final_bob -m 'Good effort but try harder next time.' + +git notes --ref=grade add setup_final_bib -m '100' +git notes --ref=feedback add setup_final_bib -m 'Perfect. But no peer review!' + +git notes --ref=feedback add setup_final_bab -m 'Please review git' + +git push origin refs/notes/*:refs/notes/* + +echo "$WORKDIR" From ad12ee4b5915852232870d815ff2ae577df30bb2 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 19:24:16 -0400 Subject: [PATCH 20/26] CI: enhance PINPing by running test-sub.sh This script the project in a more integration-test-like manner. Signed-off-by: Joel Savitz --- Containerfile | 5 ++++- start.sh | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Containerfile b/Containerfile index 087ff369..654187af 100644 --- a/Containerfile +++ b/Containerfile @@ -10,7 +10,10 @@ RUN dnf update -y && \ python-flake8 \ python-virtualenv \ python-pip \ - git + gawk \ + socat \ + git \ + git-email RUN sed -i 's/log_driver = "journald"/log_driver = "json-file"/' /usr/share/containers/containers.conf diff --git a/start.sh b/start.sh index 574df7f4..a2085be6 100755 --- a/start.sh +++ b/start.sh @@ -11,6 +11,18 @@ podman-compose logs -f submatrix 2>&1 | sed '/Synapse now listening on TCP port if [ -f test.sh ] then ./test.sh + if [ -f test-sub.sh ] + then + podman-compose down + yes | podman volume prune + podman-compose up -d + podman-compose logs -f submatrix 2>&1 | sed '/Synapse now listening on TCP port 8008/ q' + ./dev_sockets.sh & + git config --global user.name PINP + git config --global user.email podman@podman + ./test-sub.sh + + fi else virtualenv . pip install -r requirements.txt From 5729e46a89214939132222d71ae6aa78944a9dc5 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 20:08:36 -0400 Subject: [PATCH 21/26] test-sub-check.sh: add script to check results of test-sub.sh test-sub.sh is good for interactive testing, but lacks validation when automated. Add test-sub-check.sh to run some basic checks that make sure test-sub.sh ran smoothly. Signed-off-by: Joel Savitz --- script-lint.sh | 1 + start.sh | 1 + test-sub-check.sh | 118 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100755 test-sub-check.sh diff --git a/script-lint.sh b/script-lint.sh index 455ca2a9..3e259437 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -8,6 +8,7 @@ set -ex shellcheck script-lint.sh shellcheck test.sh shellcheck test-sub.sh +shellcheck test-sub-check.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/start.sh b/start.sh index a2085be6..68e886de 100755 --- a/start.sh +++ b/start.sh @@ -21,6 +21,7 @@ then git config --global user.name PINP git config --global user.email podman@podman ./test-sub.sh + ./test-sub-check.sh fi else diff --git a/test-sub-check.sh b/test-sub-check.sh new file mode 100755 index 00000000..2e38b9a6 --- /dev/null +++ b/test-sub-check.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash + +CURL_OPTS=( \ +--verbose \ +--cacert test/ca_cert.pem \ +--fail \ +--no-progress-meter \ +) + +set -exuo pipefail + +HOSTNAME_FROM_DOTENV="$(sh -c ' +set -o allexport +. ./.env +exec jq -r -n "env.SINGULARITY_HOSTNAME" +')" + +DOCKER=${DOCKER:-podman} + +SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} + +# Create test dir if it does not exist yet +mkdir -p test + +# Reset the test directory +rm -f test/* + +${DOCKER} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem + +# login as bob +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies \ + --data "username=bob&password=builder" \ + | tee test/login_success \ + | grep "msg = bob authenticated by password" + + +# check for setup assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard \ + | grep "setup" + +grep "Total Score: 64.9" test/dashboard + +grep "missing cover letter and first patch failed to apply!" test/dashboard + +grep "bib Peer Review" test/dashboard + +grep "bab Peer Review" test/dashboard + +grep "patchset applies." test/dashboard + +grep "Good effort but try harder next time." test/dashboard + +# login as bab +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies_bab \ + --data "username=bab&password=builder" \ + | tee test/login_success_bab \ + | grep "msg = bab authenticated by password" + + +# check for setup assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies_bab \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard_bab \ + | grep "setup" + +grep "Total Score: 0.0" test/dashboard_bab + +grep "bib Peer Review" test/dashboard_bab + +grep "bob Peer Review" test/dashboard_bab + +grep "patchset applies." test/dashboard_bab + +grep "illegal patch 1: permission denied for path work!" test/dashboard_bab + +grep 'Please review git' test/dashboard_bab + +# login as bib +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies_bib \ + --data "username=bib&password=builder" \ + | tee test/login_success_bib \ + | grep "msg = bib authenticated by password" + + +# check for setup assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies_bib \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard_bib \ + | grep "setup" + +grep "Total Score: 80.0" test/dashboard_bib + +grep "bob Peer Review" test/dashboard_bib + +grep "bab Peer Review" test/dashboard_bib + +grep "patchset applies." test/dashboard_bib + +grep 'Perfect. But no peer review!' test/dashboard_bib + +echo "ALL SUBMISSION TESTS PASS" From 53ee0de7b8b52277f6b5a6dd3aa35160a96238d7 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 22:01:18 -0400 Subject: [PATCH 22/26] treewide: s/DOCKER/PODMAN/g We use podman around these parts Signed-off-by: Joel Savitz --- git/admin.sh | 4 ++-- orbit/warpdrive.sh | 4 ++-- test-sub-check.sh | 4 ++-- test.sh | 16 ++++++++-------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/git/admin.sh b/git/admin.sh index f3a9e46e..bbd5189a 100755 --- a/git/admin.sh +++ b/git/admin.sh @@ -1,6 +1,6 @@ #!/bin/sh set -e -DOCKER_COMPOSE=${DOCKER_COMPOSE:-podman-compose} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} -$DOCKER_COMPOSE exec git ./create-repo.sh "$@" +$PODMAN_COMPOSE exec git ./create-repo.sh "$@" diff --git a/orbit/warpdrive.sh b/orbit/warpdrive.sh index 8d92572d..c78e8158 100755 --- a/orbit/warpdrive.sh +++ b/orbit/warpdrive.sh @@ -4,6 +4,6 @@ set -e -DOCKER_COMPOSE=${DOCKER_COMPOSE:-podman-compose} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} -$DOCKER_COMPOSE exec orbit ./hyperspace.py "$@" +$PODMAN_COMPOSE exec orbit ./hyperspace.py "$@" diff --git a/test-sub-check.sh b/test-sub-check.sh index 2e38b9a6..894de144 100755 --- a/test-sub-check.sh +++ b/test-sub-check.sh @@ -15,7 +15,7 @@ set -o allexport exec jq -r -n "env.SINGULARITY_HOSTNAME" ')" -DOCKER=${DOCKER:-podman} +PODMAN=${PODMAN:-podman} SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} @@ -25,7 +25,7 @@ mkdir -p test # Reset the test directory rm -f test/* -${DOCKER} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem # login as bob curl --url "https://$SINGULARITY_HOSTNAME/login" \ diff --git a/test.sh b/test.sh index 7e13f23f..2b5d9430 100755 --- a/test.sh +++ b/test.sh @@ -10,15 +10,15 @@ # of the furthest right failing command or zero if no command failed (o pipefail) set -exuo pipefail -DOCKER=${DOCKER:-podman} -DOCKER_COMPOSE=${DOCKER_COMPOSE:-podman-compose} +PODMAN=${PODMAN:-podman} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} require() { command -v "$1" > /dev/null || { echo "error: $1 command required yet absent" ; exit 1 ; } ; } require curl require jq require flake8 -require "${DOCKER}" -require "${DOCKER_COMPOSE}" +require "${PODMAN}" +require "${PODMAN_COMPOSE}" # Check for shell script style compliance with shellcheck ./script-lint.sh @@ -40,7 +40,7 @@ exec jq -r -n "env.SINGULARITY_HOSTNAME" SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} -${DOCKER} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem CURL_OPTS=( \ --verbose \ @@ -150,10 +150,10 @@ curl --url "pop3s://$SINGULARITY_HOSTNAME" \ orbit/warpdrive.sh -u resu -p ssap -n # Limit `resu`'s access to the empty inbox -${DOCKER_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -d resu +${PODMAN_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -d resu # Update list of email to include new message -${DOCKER_COMPOSE} exec denis /usr/local/bin/init_journal /var/lib/email/journal/journal /var/lib/email/journal/temp /var/lib/email/mail +${PODMAN_COMPOSE} exec denis /usr/local/bin/init_journal /var/lib/email/journal/journal /var/lib/email/journal/temp /var/lib/email/mail # Check that the user can get the most recent message sent to the server curl --url "pop3s://$SINGULARITY_HOSTNAME/1" \ @@ -173,7 +173,7 @@ curl --url "pop3s://$SINGULARITY_HOSTNAME" \ # Remove limit on `resu`'s access to the inbox -${DOCKER_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -a resu +${PODMAN_COMPOSE} exec denis /usr/local/bin/restrict_access /var/lib/email/journal/journal -a resu # Check that `resu` can now get the most recent message sent to the server curl --url "pop3s://$SINGULARITY_HOSTNAME/1" \ From 31cc50bb2b1921fd86d42fc8ada87812394a0703 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Fri, 30 May 2025 22:28:42 -0400 Subject: [PATCH 23/26] test: consolidate repetitive setup in test-lib test.sh and test-sub-check.sh now source test-lib script-lint.sh modified to follow source commands for those two files. Signed-off-by: Joel Savitz --- script-lint.sh | 4 ++-- test-lib | 29 +++++++++++++++++++++++++++++ test-sub-check.sh | 25 ++----------------------- test.sh | 26 ++------------------------ 4 files changed, 35 insertions(+), 49 deletions(-) create mode 100644 test-lib diff --git a/script-lint.sh b/script-lint.sh index 3e259437..0f68fbe6 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -6,9 +6,9 @@ require shellcheck set -ex shellcheck script-lint.sh -shellcheck test.sh +shellcheck -x test.sh shellcheck test-sub.sh -shellcheck test-sub-check.sh +shellcheck -x test-sub-check.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh diff --git a/test-lib b/test-lib new file mode 100644 index 00000000..a29d3963 --- /dev/null +++ b/test-lib @@ -0,0 +1,29 @@ +PODMAN=${PODMAN:-podman} +PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} + +HOSTNAME_FROM_DOTENV="$(sh -c ' +set -o allexport +. ./.env +exec jq -r -n "env.SINGULARITY_HOSTNAME" +')" + +SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} + +setup_testdir() { + # Create test dir if it does not exist yet + mkdir -p test + + # Reset the test directory + rm -f test/* + + # put the cert in there + ${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +} + +CURL_OPTS=( \ +--verbose \ +--cacert test/ca_cert.pem \ +--fail \ +--no-progress-meter \ +) + diff --git a/test-sub-check.sh b/test-sub-check.sh index 894de144..ae42ac92 100755 --- a/test-sub-check.sh +++ b/test-sub-check.sh @@ -1,31 +1,10 @@ #!/usr/bin/env bash -CURL_OPTS=( \ ---verbose \ ---cacert test/ca_cert.pem \ ---fail \ ---no-progress-meter \ -) - set -exuo pipefail -HOSTNAME_FROM_DOTENV="$(sh -c ' -set -o allexport -. ./.env -exec jq -r -n "env.SINGULARITY_HOSTNAME" -')" - -PODMAN=${PODMAN:-podman} - -SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} - -# Create test dir if it does not exist yet -mkdir -p test - -# Reset the test directory -rm -f test/* +source test-lib -${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem +setup_testdir # login as bob curl --url "https://$SINGULARITY_HOSTNAME/login" \ diff --git a/test.sh b/test.sh index 2b5d9430..c28d13da 100755 --- a/test.sh +++ b/test.sh @@ -10,8 +10,7 @@ # of the furthest right failing command or zero if no command failed (o pipefail) set -exuo pipefail -PODMAN=${PODMAN:-podman} -PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} +source test-lib require() { command -v "$1" > /dev/null || { echo "error: $1 command required yet absent" ; exit 1 ; } ; } require curl @@ -26,28 +25,7 @@ require "${PODMAN_COMPOSE}" # Check python style compliance flake8 -# Create test dir if it does not exist yet -mkdir -p test - -# Reset the test directory -rm -f test/* - -HOSTNAME_FROM_DOTENV="$(sh -c ' -set -o allexport -. ./.env -exec jq -r -n "env.SINGULARITY_HOSTNAME" -')" - -SINGULARITY_HOSTNAME=${SINGULARITY_HOSTNAME:-"${HOSTNAME_FROM_DOTENV}"} - -${PODMAN} cp singularity_nginx_1:/etc/ssl/nginx/fullchain.pem test/ca_cert.pem - -CURL_OPTS=( \ ---verbose \ ---cacert test/ca_cert.pem \ ---fail \ ---no-progress-meter \ -) +setup_testdir # Check that registration fails before user creation curl --url "https://$SINGULARITY_HOSTNAME/register" \ From cf1d3b58ff3edd38cccf48842e4d7da9979b9af5 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Sun, 1 Jun 2025 22:21:32 -0400 Subject: [PATCH 24/26] test: add more rubric tests cleanly by consolidating common testing code Add test-sub2.sh which tests out 12 invalid and 1 valid submission on a rubric for an example coding assignment and validate the resulting status values added to the email ID tags in the grading repo. Simplify the new tests by abstracting the setup section and repetitive commands, enabling quicker development of shorter future testing scripts the same flavor as the rest of test-sub*.sh. This is important as we want to make sure our automated grading features actually do what we think they will do to the fullest extent possible before this code goes into production with real students. Signed-off-by: Joel Savitz --- script-lint.sh | 5 +- start.sh | 1 + test-lib | 166 +++++++++++++++++ test-sub-check.sh | 2 - test-sub.sh | 457 ++++++++-------------------------------------- test-sub2.sh | 154 ++++++++++++++++ test.sh | 8 - 7 files changed, 400 insertions(+), 393 deletions(-) create mode 100755 test-sub2.sh diff --git a/script-lint.sh b/script-lint.sh index 0f68fbe6..2731cad9 100755 --- a/script-lint.sh +++ b/script-lint.sh @@ -5,10 +5,12 @@ require shellcheck set -ex +# -x needed to make shellcheck follow `source` command shellcheck script-lint.sh shellcheck -x test.sh -shellcheck test-sub.sh +shellcheck -x test-sub.sh shellcheck -x test-sub-check.sh +shellcheck -x test-sub2.sh shellcheck orbit/warpdrive.sh shellcheck denis/configure.sh shellcheck mailman/inspector.sh @@ -18,6 +20,5 @@ shellcheck git/setup-repo.sh shellcheck git/cgi-bin/git-receive-pack shellcheck git/hooks/post-update -# -x needed to make shellcheck follow `source` command shellcheck -x backup/backup.sh shellcheck -x backup/restore.sh diff --git a/start.sh b/start.sh index 68e886de..6743a6d5 100755 --- a/start.sh +++ b/start.sh @@ -22,6 +22,7 @@ then git config --global user.email podman@podman ./test-sub.sh ./test-sub-check.sh + ./test-sub2.sh fi else diff --git a/test-lib b/test-lib index a29d3963..1b9607fb 100644 --- a/test-lib +++ b/test-lib @@ -1,5 +1,15 @@ +# This line: +# - aborts the script after any pipeline returns nonzero (e) +# - shows all commands as they are run (x) +# - sets any dereference of an unset variable to trigger an error (u) +# - causes the return value of a pipeline to be the nonzero return value +# of the furthest right failing command or zero if no command failed (o pipefail) +set -exuo pipefail + PODMAN=${PODMAN:-podman} PODMAN_COMPOSE=${PODMAN_COMPOSE:-podman-compose} +SCRIPT_DIR=$(dirname "$(readlink -f "$0")") +WORKDIR=$(mktemp -d) HOSTNAME_FROM_DOTENV="$(sh -c ' set -o allexport @@ -27,3 +37,159 @@ CURL_OPTS=( \ --no-progress-meter \ ) + +get_git_port() { ${PODMAN} port singularity_git_1 | awk -F':' '{ print $2 }' ; } ; + +# preconditions: +# - SCRIPT_DIR defined (singularity repo root) +# - WORKDIR defined (arbitrary) temp directory +setup_submissions_and_grading_repo() { + pushd "$SCRIPT_DIR" + # nuke grading repo inside container + # shellcheck disable=SC2016 + ${PODMAN_COMPOSE} exec git ash -c 'cd /var/lib/git/grading.git && for t in $(git tag); do git tag -d $t; done' + # crete and push fresh submissions repo + rm -rf repos/submissions + git/admin.sh submissions "course submissions repository" + pushd repos + git init --bare submissions + echo "course submissions repository" > submissions/description + # create a temporary workdir to push an initial commit to the submission repo + git init submissions_init + pushd submissions_init + echo "# submissions" > README.md + git add README.md + git status + git -c user.name=singularity -c user.email=singularity@singularity commit -sm 'init submissions repo' + git push ../submissions master + popd + rm -rf submissions_init + pushd submissions + git push --mirror http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/submissions + popd + popd + popd + + pushd "$WORKDIR" + mkdir certs + pushd certs + ${PODMAN} volume export singularity_ssl-certs > certs.tar + tar xf certs.tar + popd + git clone http://localhost:"$(get_git_port)"/submissions + popd +} + +create_lighting_assignment() { + test -n "$1" + test -n "$2" + test -n "$3" + test -n "$4" + + local asn="$1" + local i_s="$2" + local p_s="$3" + local f_s="$4" + set +u + local rub="$5" + set -u + + RUBRIC= + if [ -n "$rub" ] + then + RUBRIC="-r $rub" + fi + + pushd "$SCRIPT_DIR" + # create or recreate setup assignment + if denis/configure.sh dump | grep -q "^$asn:"; then + denis/configure.sh remove -a "$asn" + fi + denis/configure.sh create -a "$asn" -i "$(date -d "$i_s secs" +%s)" -p "$(date -d "$p_s secs" +%s)" -f "$(date -d "$f_s secs" +%s)" ${RUBRIC} + denis/configure.sh reload + popd +} + +# preconditions: +# - called after setup_submissions_and_grading_repo +setup_submissions_for() { + test -n "$1" + local user="$1" + + pushd "$SCRIPT_DIR" + orbit/warpdrive.sh -u "$user" -p builder -n || orbit/warpdrive.sh -u "$user" -p builder -m + popd + + pushd "$WORKDIR"/submissions + git config user.name "$user" + git config user.email "$user"@localhost.localdomain + git config sendemail.smtpUser "$user" + git config sendemail.smtpPass builder + git config sendemail.smtpserver localhost.localdomain + git config sendemail.smtpserverport 465 + git config sendemail.smtpencryption ssl + popd +} + +# preconditions: +# - no non-tracked files in submissions repo +# - called after setup_submissions_for +enter_and_checkout() { + test -n "$1" + local branch="$1" + + pushd "$WORKDIR"/submissions + git checkout --orphan "$1" + if [ -n "$(ls)" ] + then + git rm -rf ./* + fi +} + +# preconditions: +# - called after a call to enter_and_checkout +exit_after_sending() { + test -n "$1" + local asn="$1" + + git send-email \ + --confirm=never \ + --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem \ + --to "$asn"@localhost.localdomain \ + ./*.patch + rm ./*.patch + popd + sleep 1 +} + +# preconditions: +# - called after a call to enter_and_checkout +write_commit_to() { + test -n "$1" && test -n "$2" + local content="$1" + local file="$2" + set +u # necessary since $3 is unbound in 2 arg case + local opt="$3" + set -x + + mkdir -p $(dirname "$file") + + if [ "$opt" == "append" ]; then + echo "$content" >> "$file" + else + echo "$content" > "$file" + fi + + git add "$file" + git commit -sm "add $content to $file" +} + +# preconditions: +# - called after a call to write_commit_to +fixup_cover() { + test -n "$1" + sed -i "s/\*\*\* SUBJECT HERE \*\*\*/$1/g" *0000-cover-letter.patch + sed -i "s/\*\*\* BLURB HERE \*\*\*/$1/g" *0000-cover-letter.patch + sed -i "\$a\\Signed-off-by: $(git config user.name) <$(git config user.email)>" *0000-cover-letter.patch +} + diff --git a/test-sub-check.sh b/test-sub-check.sh index ae42ac92..4f49ae3d 100755 --- a/test-sub-check.sh +++ b/test-sub-check.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -set -exuo pipefail - source test-lib setup_testdir diff --git a/test-sub.sh b/test-sub.sh index 6ddd7eaa..d02470ff 100755 --- a/test-sub.sh +++ b/test-sub.sh @@ -1,283 +1,90 @@ #!/bin/bash -set -exuo pipefail - -SCRIPT_DIR=$(dirname "$0") - -cd "$SCRIPT_DIR" - -get_git_port() { podman port singularity_git_1 | awk -F':' '{ print $2 }' ; } ; - -# nuke grading repo inside container -# shellcheck disable=SC2016 -podman-compose exec git ash -c 'cd /var/lib/git/grading.git && for t in $(git tag); do git tag -d $t; done' - -# setup submissions repo -rm -rf repos/submissions -git/admin.sh submissions "course submissions repository" -pushd repos -git init --bare submissions -echo "course submissions repository" > submissions/description -git init submissions_init -pushd submissions_init -echo "# submissions" > README.md -git add README.md -git -c user.name=singularity -c user.email=singularity@singularity commit -m 'init submissions repo' -git push ../submissions master -popd -rm -rf submissions_init -pushd submissions -git push --mirror http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/submissions -popd -popd +# assumes singularity is running when invoked +# script is written to be as idempotent as posible -# create bob,bib,bab users or change their passwords to builder -orbit/warpdrive.sh -u bob -p builder -n || orbit/warpdrive.sh -u bob -p builder -m -orbit/warpdrive.sh -u bib -p builder -n || orbit/warpdrive.sh -u bob -p builder -m -orbit/warpdrive.sh -u bab -p builder -n || orbit/warpdrive.sh -u bob -p builder -m +source test-lib -WORKDIR=$(mktemp -d) +setup_submissions_and_grading_repo cat << EOF > "$WORKDIR"/setup_rubric [{('--- /dev/null', '+++ b/bob/setup/work'): 0}, {('--- a/bob/setup/work', '+++ b/bob/setup/work'): 0}] EOF -# create or recreate setup assignment -if denis/configure.sh dump | grep -q "^setup:"; then - denis/configure.sh remove -a setup -fi -denis/configure.sh create -a setup -i "$(date -d "15 secs" +%s)" -p "$(date -d "25 secs" +%s)" -f "$(date -d "30 secs" +%s)" -r "$WORKDIR"/setup_rubric -denis/configure.sh reload - -# setup assignment environment -pushd "$WORKDIR" - -mkdir certs -pushd certs -podman volume export singularity_ssl-certs > certs.tar -tar xf certs.tar -popd -git clone http://localhost:"$(get_git_port)"/submissions - -# bob env setup - -pushd submissions -git config user.name bob -git config user.email bob@localhost.localdomain -git config sendemail.smtpUser bob -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd +create_lighting_assignment setup 15 25 30 "$WORKDIR"/setup_rubric # setup_intial_bob patchsets +setup_submissions_for bob # good patchset -pushd submissions -git checkout --orphan setup_initial_bob_good -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bob_good +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Good patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/Good patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 +fixup_cover "Good patchset" +exit_after_sending setup # corrupt patchset -pushd submissions -git checkout --orphan setup_initial_bob_corrupt -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - -echo "ghi" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add ghi to work' - +enter_and_checkout setup_initial_bob_corrupt +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" +write_commit_to "ghi" bob/setup/work "append" git format-patch --rfc --cover-letter -v1 -3 rm v1-0002-*.patch - -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Corrupt patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/Corrupt patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 +fixup_cover "Corrupt patchset" +exit_after_sending setup # patchset with whitespace errors -pushd submissions -git checkout --orphan setup_initial_bob_whitespace -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def " >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bob_whitespace +write_commit_to "abc" bob/setup/work +write_commit_to "def " bob/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Patchset with whitespace errors/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/Patchset with whitespace errors/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 +fixup_cover "Patchset with whitespace errors" +exit_after_sending setup # patchset with no cover letter -pushd submissions -git checkout --orphan setup_initial_bob_nocover -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bob_nocover +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" git format-patch --rfc -v1 -2 - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 +exit_after_sending setup # patchset with no cover letter and corrupt first patch -pushd submissions -git checkout --orphan setup_initial_bob_nocover-corrupt -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - -echo "ghi" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add ghi to work' - +enter_and_checkout setup_initial_bob_nocover-corrupt +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" +write_commit_to "ghi" bob/setup/work "append" git format-patch --rfc -v1 -2 - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd - -# end of setup_initial_bob patchsets - -# bab env setup - -pushd submissions -git config user.name bab -git config user.email bab@localhost.localdomain -git config sendemail.smtpUser bab -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd +exit_after_sending setup # setup_initial_bab submissions +setup_submissions_for bab # bab good patchset -pushd submissions -git checkout --orphan setup_initial_bab_good -git rm -rf ./* -mkdir -p bab/setup - -echo "abc" > bab/setup/work -git add bab/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bab/setup/work -git add bab/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bab_good +write_commit_to "abc" bab/setup/work +write_commit_to "def" bab/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bab: Good patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/bab: Good patchset/g' v1-0000-cover-letter.patch +fixup_cover "Good patchset" +exit_after_sending setup -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm -rf ./*.patch -popd - -# end of bab initial submission - -# bib env setup -pushd submissions -git config user.name bib -git config user.email bib@localhost.localdomain -git config sendemail.smtpUser bib -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd - -# bib initial submissions +# setup_initial_bib submissions +setup_submissions_for bib # bib good patchset -pushd submissions -git checkout --orphan setup_initial_bib_good -git rm -rf ./* -mkdir -p bib/setup - -echo "abc" > bib/setup/work -git add bib/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bib/setup/work -git add bib/setup/work -git commit -sm 'add def to work' - +enter_and_checkout setup_initial_bib_good +write_commit_to "abc" bib/setup/work +write_commit_to "def" bib/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bib: Good patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/bib: Good patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 - -# end of bib initial submission +fixup_cover "Good patchset" +exit_after_sending setup # investigate grading repo +pushd "$WORKDIR" git clone http://localhost:"$(get_git_port)"/grading.git pushd grading git fetch --tag -git tag - STATUSES=( 'patchset applies.' @@ -300,36 +107,17 @@ for t in $(git tag); do fi i=$((i+1)) done - popd -echo "wait for initial submission deadline (10 secs)" -sleep 10 - -# bob env setup - -pushd submissions -git config user.name bob -git config user.email bob@localhost.localdomain -git config sendemail.smtpUser bob -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd +echo "wait for initial submission deadline (5 secs)" +sleep 5 +setup_submissions_for bob # setup bob peer review -pushd submissions -git checkout --orphan setup_reviews_bob -git rm -rf ./* -# mkdir -p bob/setup (must be able to omit if missing) -echo ORPHAN > ORPHAN -git add ORPHAN -git commit -m 'orphan base' +enter_and_checkout setup_reviews_bob # submit peer review 1 - cat < review1 Subject: setup review 1 for bab In-Reply-To: <${PEER1_ID::-1}1@localhost.localdomain> @@ -338,14 +126,10 @@ Looks good to me Acked-by: bob PEER_REVIEW1 - git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bab@localhost.localdomain review1 rm review1 -# end peer review 1 - # submit peer review 2 - cat < review2 Subject: setup review 2 for bib In-Reply-To: <${PEER2_ID::-1}2@localhost.localdomain> @@ -354,131 +138,39 @@ Looks good to me Acked-by: bob PEER_REVIEW2 - git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to bib@localhost.localdomain review2 rm review2 -# end peer review 2 - -# end bob peer review popd sleep 1 +# end bob peer review -# bob env setup - -pushd submissions -git config user.name bob -git config user.email bob@localhost.localdomain -git config sendemail.smtpUser bob -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd - -# bob make final submission - -# good final patchset -pushd submissions -git checkout --orphan setup_final_bob_good -git rm -rf ./* -mkdir -p bob/setup - -echo "abc" > bob/setup/work -git add bob/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bob/setup/work -git add bob/setup/work -git commit -sm 'add def to work' - +# setup_final_bob good patchset +setup_submissions_for bob +enter_and_checkout setup_final_bob_good +write_commit_to "abc" bob/setup/work +write_commit_to "def" bob/setup/work "append" git format-patch --cover-letter -v2 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/Good patchset final/g' v2-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/Good patchset final/g' v2-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 - -# end bob final submission - -# bib env setup - -pushd submissions -git config user.name bib -git config user.email bib@localhost.localdomain -git config sendemail.smtpUser bib -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd - -# bib final submission labeled RFC -pushd submissions -git checkout --orphan setup_final_bib_rfc -git rm -rf ./* -mkdir -p bib/setup - -echo "abc" > bib/setup/work -git add bib/setup/work -git commit -sm 'add abc to work' - -echo "def" >> bib/setup/work -git add bib/setup/work -git commit -sm 'add def to work' - +fixup_cover "Good final patchset" +exit_after_sending setup + +# setup_final_bib final submission incorrectly labeled RFC +setup_submissions_for bib +enter_and_checkout setup_final_bib_rfc +write_commit_to "abc" bib/setup/work +write_commit_to "def" bib/setup/work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bib: RFC patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/bib: RFC patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 - -# end of bib final submission labeled RFC - - -# bab env setup - -pushd submissions -git config user.name bab -git config user.email bab@localhost.localdomain -git config sendemail.smtpUser bab -git config sendemail.smtpPass builder -git config sendemail.smtpserver localhost.localdomain -git config sendemail.smtpserverport 465 -git config sendemail.smtpencryption ssl -popd - - -# bab final submission in wrong directory -pushd submissions -git checkout --orphan setup_final_bab_illegal -git rm -rf ./* -mkdir -p bib/setup - -echo "abc" > work -git add work -git commit -sm 'add abc to work' - -echo "def" >> work -git add work -git commit -sm 'add def to work' - +fixup_cover "RFC tagged final patchset" +exit_after_sending setup + +# setup_final_bab submission in wrong directory +setup_submissions_for bab +enter_and_checkout setup_final_bab_illegal +write_commit_to "abc" work +write_commit_to "def" work "append" git format-patch --rfc --cover-letter -v1 -2 -sed -i 's/\*\*\* SUBJECT HERE \*\*\*/bab: Illegal patchset/g' v1-0000-cover-letter.patch -sed -i 's/\*\*\* BLURB HERE \*\*\*/bab: Illegal patchset/g' v1-0000-cover-letter.patch - -git send-email --confirm=never --smtp-ssl-cert-path="$WORKDIR"/certs/fullchain.pem --to setup@localhost.localdomain ./*.patch -rm ./*.patch -popd -sleep 1 - -# end of bab final submission in wrong directory - +fixup_cover "Illegal patchset" +exit_after_sending setup echo "wait for final submission deadline (10 secs)" sleep 10 @@ -488,14 +180,17 @@ git fetch --tag sleep 2 git fetch -f origin refs/notes/*:refs/notes/* git remote set-url --push origin http://localhost:"$(get_git_port)"/cgi-bin/git-receive-pack/grading.git + git notes --ref=grade add setup_final_bob -m '66' git notes --ref=grade add setup_review1_bob -m '22' git notes --ref=grade add setup_review2_bob -m '99' git notes --ref=feedback add setup_final_bob -m 'Good effort but try harder next time.' +# automatic 0 for peer reviews git notes --ref=grade add setup_final_bib -m '100' git notes --ref=feedback add setup_final_bib -m 'Perfect. But no peer review!' +# automatic 0 for peer reviews and final submission git notes --ref=feedback add setup_final_bab -m 'Please review git' git push origin refs/notes/*:refs/notes/* diff --git a/test-sub2.sh b/test-sub2.sh new file mode 100755 index 00000000..80e5e733 --- /dev/null +++ b/test-sub2.sh @@ -0,0 +1,154 @@ +#!/bin/bash + +# test a bunch of rubric violations + +source test-lib + +setup_submissions_and_grading_repo + +cat << EOF > "$WORKDIR"/coding_rubric +[{('--- /dev/null', '+++ b/user/coding/program.c'): 0}, +{('--- a/user/coding/program.c', '+++ b/user/coding/program.c'): 0}, +{('--- /dev/null', '+++ b/user/coding/Makefile'): 0}] +EOF + +create_lighting_assignment coding 25 28 30 "$WORKDIR"/coding_rubric + +setup_submissions_for bob + +enter_and_checkout coding_initial_bob_toomany +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Yet more C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -4 +fixup_cover "Too many patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_toofew +write_commit_to "C program" bob/coding/program.c +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -2 +fixup_cover "Too few patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_top-level-first +write_commit_to "C program" program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Top level first file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_top-level-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Top level third file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_second-level-first +write_commit_to "C program" bob/program.c +write_commit_to "More C program" bob/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Second level first file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_second-level-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Second level third file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_fourth-level-first +write_commit_to "C program" bob/coding/program/program.c +write_commit_to "More C program" bob/coding/program/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Fourth level first file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_fourth-level-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/program/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Fourth level third file patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-second-dir-first +write_commit_to "C program" bob/setup/program.c +write_commit_to "More C program" bob/setup/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong second directory first patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-second-dir-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/setup/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong second directory third patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-filename-first +write_commit_to "C program" bob/coding/hello.c +write_commit_to "More C program" bob/coding/hello.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong filename first patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_wrong-filename-third +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/Snakefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Wrong filename third patches" +exit_after_sending coding + +enter_and_checkout coding_initial_bob_good +write_commit_to "C program" bob/coding/program.c +write_commit_to "More C program" bob/coding/program.c "append" +write_commit_to "Makefile content" bob/coding/Makefile +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Good patches" +exit_after_sending coding + +pushd "$WORKDIR" +git clone http://localhost:"$(get_git_port)"/grading.git +pushd grading + +STATUSES=( +"patch count 4 violates expected rubric patch count of 3!" +"patch count 2 violates expected rubric patch count of 3!" +"illegal patch 1: permission denied for path program.c!" +"illegal patch 3: permission denied for path Makefile!" +"patch 1 violates the assignment rubric!" +"patch 3 violates the assignment rubric!" +"patch 1 violates the assignment rubric!" +"patch 3 violates the assignment rubric!" +"patch 1 violates the assignment rubric!" +"patch 3 violates the assignment rubric!" +"patch 1 violates the assignment rubric!" +"patch 3 violates the assignment rubric!" +"patchset applies." +) + +# submitted patchses should have statuses in this order +# assumption: first ID tag is first patch submitted by this script and IDs increase monotonically +i=0 +for t in $(git tag); do + git show -s --oneline "$t" | grep -q "${STATUSES[$i]}" + i=$((i+1)) +done +popd +popd + +echo "RUBRIC CHECKS PASS" +echo "$WORKDIR" diff --git a/test.sh b/test.sh index c28d13da..f87528c9 100755 --- a/test.sh +++ b/test.sh @@ -2,14 +2,6 @@ # Testing script for singularity and orbit -# This line: -# - aborts the script after any pipeline returns nonzero (e) -# - shows all commands as they are run (x) -# - sets any dereference of an unset variable to trigger an error (u) -# - causes the return value of a pipeline to be the nonzero return value -# of the furthest right failing command or zero if no command failed (o pipefail) -set -exuo pipefail - source test-lib require() { command -v "$1" > /dev/null || { echo "error: $1 command required yet absent" ; exit 1 ; } ; } From 140c3f52102ef616715f9d4141866d567dffdc04 Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Mon, 2 Jun 2025 14:52:11 -0400 Subject: [PATCH 25/26] denis,mailman,orbit: rename Gradable.{status => auto_feedback} Submissions.status has the same field name and refers to a different type of message so rename this one to better reflect its content. Also update some variable names to reflect this change. Signed-off-by: Joel Savitz --- denis/utilities.py | 10 +++++----- mailman/db.py | 2 +- mailman/inspector.py | 2 +- mailman/patchset.py | 14 +++++++------- mailman/submit.py | 8 ++++---- orbit/radius.py | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/denis/utilities.py b/denis/utilities.py index 529ee753..d6bd453d 100644 --- a/denis/utilities.py +++ b/denis/utilities.py @@ -50,7 +50,7 @@ def configure_repo(repo): def update_tags(assignment, component): grd_tbl = mailman.db.Gradeable - subs = (grd_tbl.select() + gbls = (grd_tbl.select() .order_by(-grd_tbl.timestamp) .where(grd_tbl.assignment == assignment) .where(grd_tbl.component == component)) @@ -70,12 +70,12 @@ def update_tags(assignment, component): print('Potential issue? Attempted to create duplicate tag ' f'{new_tag_name}') continue - user_sub = subs.where(grd_tbl.user == user.username).first() - if not user_sub or (id := user_sub.submission_id) not in repo.tags: + user_gbl = gbls.where(grd_tbl.user == user.username).first() + if not user_gbl or (id := user_gbl.submission_id) not in repo.tags: msg = 'No gradeable submission' to_promote = repo.tags['EMPTY'] else: - msg = user_sub.status + msg = user_gbl.auto_feedback to_promote = repo.tags[id] repo.create_tag(new_tag_name, ref=to_promote.commit, message=msg) @@ -98,7 +98,7 @@ def check_corrupt_or_missing(repo, tag): msg += '\n' msg += '------------------------------' msg += '\n\n' - if not gradable or gradable.status[-1] == '!': + if not gradable or gradable.auto_feedback[-1] == '!': repo.git.execute(['git', 'notes', '--ref=grade', 'add', tag, '-m', '0']) if not gradable: msg += 'automatic 0 due to missing submission!' diff --git a/mailman/db.py b/mailman/db.py index 4b48ace5..978b351e 100755 --- a/mailman/db.py +++ b/mailman/db.py @@ -27,7 +27,7 @@ class Gradeable(BaseModel): user = peewee.TextField() assignment = peewee.TextField() component = peewee.TextField() - status = peewee.TextField(null=True) + auto_feedback = peewee.TextField(null=True) if __name__ == '__main__': diff --git a/mailman/inspector.py b/mailman/inspector.py index 95dce1a1..69da6fe2 100755 --- a/mailman/inspector.py +++ b/mailman/inspector.py @@ -78,7 +78,7 @@ def gradables(assignment, username, component): for gbl in query: print(gbl.submission_id, datetime.fromtimestamp(gbl.timestamp).astimezone().isoformat(), - gbl.assignment, gbl.user, gbl.component, gbl.status) + gbl.assignment, gbl.user, gbl.component, gbl.auto_feedback) def missing(assignment): diff --git a/mailman/patchset.py b/mailman/patchset.py index 92a9be04..ae9ebf45 100644 --- a/mailman/patchset.py +++ b/mailman/patchset.py @@ -143,20 +143,20 @@ def check(cover_letter, patches, submission_id, asn): with repo.config_writer() as config: config.set_value('user', 'name', 'mailman') config.set_value('user', 'email', 'mailman@mailman') - status = do_check(repo, cover_letter, patches, asn) - if status[-1] == '!': + auto_feedback = do_check(repo, cover_letter, patches, asn) + if auto_feedback[-1] == '!': for patch in patches: patch_abspath = str(maildir / patch.msg_id) repo.git.execute(['git', 'commit', '--allow-empty', '-F', patch_abspath]) - tag_and_push(repo, submission_id, msg=status) - return status + tag_and_push(repo, submission_id, msg=auto_feedback) + return auto_feedback def apply_peer_review(email, submission_id, review_id): args = [*git_am_args, '--empty=keep'] patch_abspath = str(maildir / email.msg_id) - status = 'sucessfully stored peer review' + auto_feedback = 'sucessfully stored peer review' with tempfile.TemporaryDirectory() as repo_path: try: @@ -171,6 +171,6 @@ def apply_peer_review(email, submission_id, review_id): tag_and_push(repo, submission_id) except git.GitCommandError as e: print(e, file=sys.stderr) - status = 'failed to apply peer review' + auto_feedback = 'failed to apply peer review' - return status + return auto_feedback diff --git a/mailman/submit.py b/mailman/submit.py index 41555bcf..9f52bd12 100755 --- a/mailman/submit.py +++ b/mailman/submit.py @@ -65,7 +65,7 @@ def set_status(status): gr_db = db.Gradeable if asn := asn_db.get_or_none(asn_db.name == emails[0].rcpt): cover_letter, *patches = emails - status = patchset.check(cover_letter, patches, logfile, asn) + auto_feedback = patchset.check(cover_letter, patches, logfile, asn) if len(emails) < 2: return set_status('missing patches') typ = ('initial' if timestamp < asn.initial_due_date @@ -73,11 +73,11 @@ def set_status(status): if not typ: return set_status(f'{asn.name} past due') gr_db.create(submission_id=logfile, timestamp=timestamp, user=user, - assignment=asn.name, component=typ, status=status) + assignment=asn.name, component=typ, auto_feedback=auto_feedback) return set_status(f'{asn.name}: {typ}') if reply_id: - status = patchset.apply_peer_review(emails[0], logfile, reply_id) + auto_feedback = patchset.apply_peer_review(emails[0], logfile, reply_id) if not (orig := gr_db.get_or_none(gr_db.submission_id == reply_id)): return set_status('not a reply to a submission') asn_name = orig.assignment @@ -100,7 +100,7 @@ def set_status(status): return set_status('reviewed wrong submission') gr_db.create(submission_id=logfile, timestamp=timestamp, user=user, assignment=asn_name, component=typ, - status=status) + auto_feedback=auto_feedback) return set_status(f'{asn_name}: {typ}') return set_status('Not a recognized recipient') diff --git a/orbit/radius.py b/orbit/radius.py index d1b7e798..e9e7a6de 100644 --- a/orbit/radius.py +++ b/orbit/radius.py @@ -464,9 +464,9 @@ def get_automated_feedback(self, attr): due_date = int(self.assignment.final_due_date) if due_date < int(datetime.now().timestamp()): - return gbl.status + return gbl.auto_feedback - match gbl.status[-1]: + match gbl.auto_feedback[-1]: case '.': return 'Submission accepted' case '?': From 334651c981443739714030fc6448867c1ed3387d Mon Sep 17 00:00:00 2001 From: Joel Savitz Date: Mon, 2 Jun 2025 15:02:54 -0400 Subject: [PATCH 26/26] test: add test for whether only initial submission gets autozero If a student makes an inital submission but not a final submission, they should automatically receive a zero for the assignment. Signed-off-by: Joel Savitz --- start.sh | 7 +++++++ test-sub3.sh | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100755 test-sub3.sh diff --git a/start.sh b/start.sh index 6743a6d5..504c0028 100755 --- a/start.sh +++ b/start.sh @@ -22,7 +22,14 @@ then git config --global user.email podman@podman ./test-sub.sh ./test-sub-check.sh + podman-compose down + yes | podman volume prune + podman-compose up -d ./test-sub2.sh + podman-compose down + yes | podman volume prune + podman-compose up -d + ./test-sub3.sh fi else diff --git a/test-sub3.sh b/test-sub3.sh new file mode 100755 index 00000000..55b28c30 --- /dev/null +++ b/test-sub3.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# test only initial submission results in automatic 0 + +source test-lib + +setup_submissions_and_grading_repo + +create_lighting_assignment fun 5 7 10 + +setup_submissions_for bob + +enter_and_checkout fun_initial_bob_only-initial +write_commit_to "C program" bob/fun/program.c +write_commit_to "More C program" bob/fun/program.c "append" +write_commit_to "Yet more C program" bob/fun/program.c "append" +git format-patch -v1 --cover-letter --rfc -3 +fixup_cover "Only initial sub patchset" +exit_after_sending fun + +# wait for initial submission deadline +sleep 9 + +setup_testdir + +# login as bob +curl --url "https://$SINGULARITY_HOSTNAME/login" \ + --unix-socket ./socks/https.sock \ + "${CURL_OPTS[@]}" \ + -c test/cookies \ + --data "username=bob&password=builder" \ + | tee test/login_success \ + | grep "msg = bob authenticated by password" + + +# check for fun assignment and save dashboard +curl --url "https://$SINGULARITY_HOSTNAME/dashboard" \ + --unix-socket ./socks/https.sock \ + -b test/cookies \ + "${CURL_OPTS[@]}" \ + | tee test/dashboard \ + | grep "fun" + +grep "patchset applies." test/dashboard +grep "No submission" test/dashboard +grep "Total Score: 0.0" test/dashboard + +echo "INITIAL SUBMISSION ONLY GETS AUTO ZERO CONFIRMED"

{self.name}
Total Score: -Total Score: {self.get_total_score()} Timestamp Submission ID Request an 'Oopsie'Automated Feedback {self.get_automated_feedback('final')}
Human Feedback{self.get_human_feedback()}
Automated Feedback {self.get_automated_feedback('final')}
Human Feedback{self.get_human_feedback()}