From 3862c47eccd9b666521b975e98f1ca69a026f21b Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Sat, 16 May 2026 23:19:40 +0100 Subject: [PATCH 01/16] fix(api): improve CNAME exclusivity error message --- api/desecapi/serializers/records.py | 4 ++-- api/desecapi/tests/test_rrsets_bulk.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/desecapi/serializers/records.py b/api/desecapi/serializers/records.py index 82b8457f0..c7e6ea674 100644 --- a/api/desecapi/serializers/records.py +++ b/api/desecapi/serializers/records.py @@ -239,7 +239,7 @@ def to_internal_value(self, data): raise serializers.ValidationError( { api_settings.NON_FIELD_ERRORS_KEY: [ - f"RRset with conflicting type present: {types_by_position}." + f"RRset with conflicting type present at same subname: {types_by_position}." " (No other RRsets are allowed alongside CNAME.)" ] } @@ -412,7 +412,7 @@ def get_validators(self): "type", "CNAME", ), - message="RRset with conflicting type present: database ({types})." + message="RRset with conflicting type present at same subname: database ({types})." " (No other RRsets are allowed alongside CNAME.)", ), ] diff --git a/api/desecapi/tests/test_rrsets_bulk.py b/api/desecapi/tests/test_rrsets_bulk.py index 90c08c1e7..6f5ffc9ea 100644 --- a/api/desecapi/tests/test_rrsets_bulk.py +++ b/api/desecapi/tests/test_rrsets_bulk.py @@ -238,12 +238,12 @@ def test_bulk_patch_cname_exclusivity(self): [ { "non_field_errors": [ - "RRset with conflicting type present: 1 (CNAME). (No other RRsets are allowed alongside CNAME.)" + "RRset with conflicting type present at same subname: 1 (CNAME). (No other RRsets are allowed alongside CNAME.)" ] }, { "non_field_errors": [ - "RRset with conflicting type present: 0 (A), database (A, TXT). (No other RRsets are allowed alongside CNAME.)" + "RRset with conflicting type present at same subname: 0 (A), database (A, TXT). (No other RRsets are allowed alongside CNAME.)" ] }, ], From 507d2596fa12b20a22fa916315782d8a120c976c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 04:03:11 +0000 Subject: [PATCH 02/16] chore(deps): update psycopg[binary] requirement in /api Updates the requirements on [psycopg[binary]](https://github.com/psycopg/psycopg) to permit the latest version. - [Changelog](https://github.com/psycopg/psycopg/blob/master/docs/news.rst) - [Commits](https://github.com/psycopg/psycopg/compare/3.3.2...3.3.4) --- updated-dependencies: - dependency-name: psycopg[binary] dependency-version: 3.3.4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/requirements.txt b/api/requirements.txt index 24e63dd94..ec314d65e 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -11,7 +11,7 @@ django-pgtrigger~=4.17.0 django-prometheus~=2.4.1 dnspython~=2.8.0 pyotp~=2.9.0 -psycopg[binary]~=3.3.2 +psycopg[binary]~=3.3.4 psl-dns~=1.1.1 pylibmc~=1.6.3 pyyaml~=6.0.3 From d382284caaa1abc479d3ddd83c0bf70aefd443d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 04:03:14 +0000 Subject: [PATCH 03/16] chore(deps): update responses requirement in /api Updates the requirements on [responses](https://github.com/getsentry/responses) to permit the latest version. - [Release notes](https://github.com/getsentry/responses/releases) - [Changelog](https://github.com/getsentry/responses/blob/master/CHANGES) - [Commits](https://github.com/getsentry/responses/compare/0.25.8...0.26.0) --- updated-dependencies: - dependency-name: responses dependency-version: 0.26.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/requirements.txt b/api/requirements.txt index ec314d65e..5bb3ef5dc 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -16,5 +16,5 @@ psl-dns~=1.1.1 pylibmc~=1.6.3 pyyaml~=6.0.3 requests~=2.32.5 -responses~=0.25.8 +responses~=0.26.0 uwsgi~=2.0.31 From 17473e498a6af1c68dd5e193fa9f1bb27958e2de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 04:03:25 +0000 Subject: [PATCH 04/16] chore(deps): update django-netfields requirement in /api Updates the requirements on [django-netfields](https://github.com/jimfunk/django-postgresql-netfields) to permit the latest version. - [Changelog](https://github.com/jimfunk/django-postgresql-netfields/blob/master/CHANGELOG) - [Commits](https://github.com/jimfunk/django-postgresql-netfields/compare/v1.3.2...v1.4.1) --- updated-dependencies: - dependency-name: django-netfields dependency-version: 1.4.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/requirements.txt b/api/requirements.txt index 5bb3ef5dc..9fe0210ae 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -6,7 +6,7 @@ Django~=5.2.7 django-cors-headers~=4.9.0 djangorestframework~=3.16.1 django-celery-email~=3.0.0 -django-netfields~=1.3.2 +django-netfields~=1.4.1 django-pgtrigger~=4.17.0 django-prometheus~=2.4.1 dnspython~=2.8.0 From cf4afa9aa150e90420ec26dae12db02940da35b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Mar 2026 04:03:19 +0000 Subject: [PATCH 05/16] chore(deps): update coverage requirement in /api Updates the requirements on [coverage](https://github.com/coveragepy/coveragepy) to permit the latest version. - [Release notes](https://github.com/coveragepy/coveragepy/releases) - [Changelog](https://github.com/coveragepy/coveragepy/blob/main/CHANGES.rst) - [Commits](https://github.com/coveragepy/coveragepy/compare/7.13.4...7.14.0) --- updated-dependencies: - dependency-name: coverage dependency-version: 7.14.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- api/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/requirements.txt b/api/requirements.txt index 9fe0210ae..16de56b07 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,6 +1,6 @@ captcha~=0.7.1 celery~=5.6.2 -coverage~=7.13.4 +coverage~=7.14.0 cryptography~=46.0.5 Django~=5.2.7 django-cors-headers~=4.9.0 From 9eea2feba7bd942412ad64d56a2023d78f32487e Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Sun, 24 May 2026 18:37:25 +0200 Subject: [PATCH 06/16] fix(api): WSGI_APPLICATION path --- api/api/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/api/settings.py b/api/api/settings.py index d69ddb606..5508791a8 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -52,7 +52,7 @@ ROOT_URLCONF = "api.urls" -WSGI_APPLICATION = "desecapi.wsgi.application" +WSGI_APPLICATION = "api.wsgi.application" DATABASES = { From 3723e1f9b74d94825d01472f1be0cdcaffb321c8 Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Sun, 24 May 2026 18:52:58 +0200 Subject: [PATCH 07/16] fix(api): fix overload from parallel slow requests in serials/ --- api/desecapi/views/domains.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/api/desecapi/views/domains.py b/api/desecapi/views/domains.py index a535467ee..37ce115fc 100644 --- a/api/desecapi/views/domains.py +++ b/api/desecapi/views/domains.py @@ -3,7 +3,7 @@ from django.conf import settings from django.core.cache import cache from django.db.models import Subquery -from rest_framework import mixins, viewsets +from rest_framework import mixins, status, viewsets from rest_framework.decorators import action from rest_framework.permissions import IsAuthenticated, SAFE_METHODS from rest_framework.response import Response @@ -143,8 +143,17 @@ class SerialListView(APIView): def get(self, request, *args, **kwargs): key = "desecapi.views.serials" - serials = cache.get(key) - if serials is None: + # Determine if the last update is older than 60s and nobody is working on it + # If so, start working on it; other workers will keep using the old list + our_update = cache.add(f"{key}.sentinel", True, timeout=60) + if our_update: serials = get_serials() - cache.get_or_set(key, serials, timeout=60) + cache.set(key, serials, timeout=300) # long expiration, overwritten 1/min + else: + serials = cache.get(key) + if serials is None: + return Response( + data={"detail": "Serial cache not ready"}, + status=status.HTTP_409_CONFLICT, + ) return Response(serials) From 9906552e54de2d76c5ac24d19a33058b157c0c9e Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Sun, 24 May 2026 19:33:08 +0200 Subject: [PATCH 08/16] fix(api): KeyError in d5d7768e516 --- api/desecapi/tests/test_dyndns12update.py | 10 ++++++++++ api/desecapi/views/dyndns.py | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/api/desecapi/tests/test_dyndns12update.py b/api/desecapi/tests/test_dyndns12update.py index 528ed0b77..da426acaa 100644 --- a/api/desecapi/tests/test_dyndns12update.py +++ b/api/desecapi/tests/test_dyndns12update.py @@ -161,6 +161,16 @@ def test_ddclient_dyndns2_v4_invalid(self): self.assertStatus(response, status.HTTP_400_BAD_REQUEST) self.assertIn("malformed", str(response.data)) + def test_ddclient_dyndns2_v4_weird(self): + # These should not throw errors + url = self.reverse("v1:dyndns12update") + for query_string in [ + "/https:/update.dedyn.io/example.dedyn.io", + "/&myipv4=%3Cpreserve%3E&myipv6=2a00::/64&:8:a:5:8b61", + ]: + response = self.client.get(f"{url}?{query_string}") + self.assertStatus(response, status.HTTP_200_OK) + def test_ddclient_dyndns2_v4_valid_priority(self): params = { "domain_name": self.my_domain.name, diff --git a/api/desecapi/views/dyndns.py b/api/desecapi/views/dyndns.py index c289a9188..2f47658a4 100644 --- a/api/desecapi/views/dyndns.py +++ b/api/desecapi/views/dyndns.py @@ -238,7 +238,10 @@ def extra_qname_params(self) -> dict[str, dict[str, str]]: param_prefix, param_suffix = param.split(":", 1) except ValueError: continue - type_ = param_prefix_types[param_prefix] + try: + type_ = param_prefix_types[param_prefix] + except KeyError: + continue for qname in self._sanitize_qnames(param_suffix): qnames[qname][type_] = value From f55af2da58eb95875d546ada7e31aa1f14af15f8 Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Tue, 26 May 2026 09:43:44 +0200 Subject: [PATCH 09/16] fix(nsmaster): remove TSIG key and dynamic also-notify (Step 0) - nsmaster/entrypoint.sh: remove 'pdnsutil import-tsig-key' call; the DESECSTACK_NSMASTER_TSIGKEY variable is unused (no external secondaries currently require TSIG authentication) - nsmaster/conf/pdns.conf.var: drop ',${DESECSTACK_NSMASTER_ALSO_NOTIFY}' from also-notify; hardcoded multicast address 239.1.2.3 is kept - api/desecapi/pdns.py: remove master_tsig_key_ids from create_zone_master() since the key no longer exists in nsmaster - docker-compose.yml: remove DESECSTACK_NSMASTER_TSIGKEY and DESECSTACK_NSMASTER_ALSO_NOTIFY from nsmaster service environment Co-Authored-By: Claude Sonnet 4.6 --- api/desecapi/pdns.py | 1 - docker-compose.yml | 2 -- nsmaster/conf/pdns.conf.var | 2 +- nsmaster/entrypoint.sh | 3 --- 4 files changed, 1 insertion(+), 7 deletions(-) diff --git a/api/desecapi/pdns.py b/api/desecapi/pdns.py index 7508a6a68..4eaf58bf1 100644 --- a/api/desecapi/pdns.py +++ b/api/desecapi/pdns.py @@ -253,7 +253,6 @@ def create_zone_master(name): "name": name, "kind": "SLAVE", "masters": [gethostbyname_cached("nslord")], - "master_tsig_key_ids": ["default"], }, ) diff --git a/docker-compose.yml b/docker-compose.yml index ea0f958b2..e4883849e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -206,11 +206,9 @@ services: environment: - DESECSTACK_IPV4_REAR_PREFIX16 - DESECSTACK_DBMASTER_PASSWORD_pdns - - DESECSTACK_NSMASTER_ALSO_NOTIFY - DESECSTACK_NSMASTER_APIKEY - DESECSTACK_NSMASTER_CARBONSERVER - DESECSTACK_NSMASTER_CARBONOURNAME - - DESECSTACK_NSMASTER_TSIGKEY depends_on: - dbmaster networks: diff --git a/nsmaster/conf/pdns.conf.var b/nsmaster/conf/pdns.conf.var index c74c92e40..6aa3803d3 100644 --- a/nsmaster/conf/pdns.conf.var +++ b/nsmaster/conf/pdns.conf.var @@ -1,7 +1,7 @@ api=yes api-key=${DESECSTACK_NSMASTER_APIKEY} allow-axfr-ips=10.8.0.0/24 -also-notify=239.1.2.3,${DESECSTACK_NSMASTER_ALSO_NOTIFY} +also-notify=239.1.2.3 only-notify= setgid=pdns setuid=pdns diff --git a/nsmaster/entrypoint.sh b/nsmaster/entrypoint.sh index 8ff97fbb8..48edb20ad 100755 --- a/nsmaster/entrypoint.sh +++ b/nsmaster/entrypoint.sh @@ -17,7 +17,4 @@ done # Manage credentials envsubst < /etc/powerdns/pdns.conf.var > /etc/powerdns/pdns.conf -echo "Provisioning default TSIG key ..." -pdnsutil import-tsig-key default hmac-sha256 "${DESECSTACK_NSMASTER_TSIGKEY}" > /dev/null - exec pdns_server --daemon=no From 164243e7ff4b5f0f8af8583da13c8dcd4601eaf0 Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Tue, 26 May 2026 09:45:30 +0200 Subject: [PATCH 10/16] =?UTF-8?q?feat(nsmaster):=20add=20zone=20export/imp?= =?UTF-8?q?ort=20scripts=20for=20PowerDNS=E2=86=92Knot=20migration=20(Step?= =?UTF-8?q?=201)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit export-zones.sh: runs inside the existing PowerDNS nsmaster container; uses pdnsutil to dump every zone to bind-format zone files and writes a manifest. catalog.internal is skipped — Knot generates it natively. import-zones.sh: runs inside the new Knot nsmaster container; registers all zones in a single knotc conf-begin/conf-commit transaction from the manifest, pre-loads the exported zone files, and triggers a reload. catalog.internal is not imported as it is declared statically in knot.conf with catalog-role: generate. Co-Authored-By: Claude Sonnet 4.6 --- nsmaster/export-zones.sh | 22 +++++++++++++++++++++ nsmaster/import-zones.sh | 41 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100755 nsmaster/export-zones.sh create mode 100755 nsmaster/import-zones.sh diff --git a/nsmaster/export-zones.sh b/nsmaster/export-zones.sh new file mode 100755 index 000000000..81158478b --- /dev/null +++ b/nsmaster/export-zones.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Export all zones from the PowerDNS nsmaster to /zones-export/ +# Usage: docker compose exec nsmaster /root/export-zones.sh [output-dir] +set -euo pipefail + +OUTPUT_DIR="${1:-/zones-export}" +mkdir -p "$OUTPUT_DIR" + +echo "Fetching zone list via pdnsutil ..." +ZONE_COUNT=0 + +for ZONE in $(pdnsutil list-all-zones); do + # Skip catalog.internal — Knot generates it automatically from its zone list + [[ "$ZONE" == "catalog.internal." ]] && continue + + pdnsutil export-zone "$ZONE" > "$OUTPUT_DIR/${ZONE}zone" + echo "$ZONE" >> "$OUTPUT_DIR/zone-list.txt" + ZONE_COUNT=$((ZONE_COUNT + 1)) + echo " Exported: $ZONE" +done + +echo "Exported $ZONE_COUNT zones to $OUTPUT_DIR" diff --git a/nsmaster/import-zones.sh b/nsmaster/import-zones.sh new file mode 100755 index 000000000..3261b5008 --- /dev/null +++ b/nsmaster/import-zones.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Import zones exported from PowerDNS into Knot DNS (config DB mode) +# Usage: docker compose exec nsmaster /root/import-zones.sh [input-dir] +set -euo pipefail + +INPUT_DIR="${1:-/zones-export}" +ZONE_DIR="/var/lib/knot/zones" + +if [[ ! -f "$INPUT_DIR/zone-list.txt" ]]; then + echo "ERROR: $INPUT_DIR/zone-list.txt not found" >&2 + exit 1 +fi + +mkdir -p "$ZONE_DIR" +chown knot:knot "$ZONE_DIR" + +echo "Registering zones in Knot config DB ..." + +# Open a single transaction for all zones for efficiency +knotc conf-begin + +while IFS= read -r ZONE; do + [[ -z "$ZONE" ]] && continue + + # Pre-load zone file so Knot does not have to AXFR every zone from nslord on first start + cp "$INPUT_DIR/${ZONE}zone" "$ZONE_DIR/${ZONE}zone" + chown knot:knot "$ZONE_DIR/${ZONE}zone" + + knotc conf-set "zone[$ZONE]" + knotc conf-set "zone[$ZONE].template" "slave" + knotc conf-set "zone[$ZONE].file" "$ZONE_DIR/${ZONE}zone" + + echo " Registered: $ZONE" +done < "$INPUT_DIR/zone-list.txt" + +knotc conf-commit + +echo "Reloading all zones ..." +knotc zone-reload + +echo "Import complete." From 53e977e8c099597b15185667d945e1fa9e953461 Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Tue, 26 May 2026 09:55:56 +0200 Subject: [PATCH 11/16] =?UTF-8?q?feat(nsmaster):=20replace=20PowerDNS=20wi?= =?UTF-8?q?th=20Knot=20DNS=20container=20(Steps=204a=E2=80=934c)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dockerfile: - Switch base image from ubuntu:jammy (PowerDNS) to debian:bookworm-slim - Install Knot DNS from the CZ.NIC vendor apt repository (knot, knot-dnsutils) - Include iproute2, iptables, gettext-base (same runtime deps as before) - libknot Python package is NOT installed here — it belongs in the api container which is the only service calling the control socket conf/knot.conf.var (new): - server: listen on 0.0.0.0/:: port 53; config-db at /var/lib/knot/confdb - acl: allow 10.8.0.0/24 (VPN secondaries) to transfer - remote nslord: ${DESECSTACK_IPV4_REAR_PREFIX16}.1.11@53 - remote notify-multicast: 239.1.2.3@53 (preserved from old also-notify) - template slave: master=nslord, notify=multicast, catalog-zone=catalog.internal. zonefile-sync=-1 (journal-only), zonefile-load=whole (bootstrap from file) - zone catalog.internal.: catalog-role=generate (auto-populated by Knot) conf/pdns.conf.var: removed (no longer referenced by Dockerfile) entrypoint.sh: - Remove PostgreSQL readiness wait and TSIG provisioning - Use envsubst to render knot.conf.var → /etc/knot/knot.conf - Create /var/lib/knot and /run/knot with correct ownership - chmod 777 /run/knot so the api container can reach the control socket - umask 0 before exec knotd so the socket is created world-accessible Co-Authored-By: Claude Sonnet 4.6 --- nsmaster/Dockerfile | 51 ++++++++++++++++++++----------------- nsmaster/conf/knot.conf.var | 46 +++++++++++++++++++++++++++++++++ nsmaster/conf/pdns.conf.var | 26 ------------------- nsmaster/entrypoint.sh | 35 ++++++++++++++----------- 4 files changed, 94 insertions(+), 64 deletions(-) create mode 100644 nsmaster/conf/knot.conf.var delete mode 100644 nsmaster/conf/pdns.conf.var diff --git a/nsmaster/Dockerfile b/nsmaster/Dockerfile index 860ec087a..5e82c5a39 100644 --- a/nsmaster/Dockerfile +++ b/nsmaster/Dockerfile @@ -1,29 +1,34 @@ -FROM ubuntu:jammy +FROM debian:bookworm-slim -COPY ./entrypoint.sh /root/ -CMD ["/root/entrypoint.sh"] +# Install prerequisites for adding the vendor apt repo +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + gnupg \ + ca-certificates \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Add CZ.NIC vendor repository for Knot DNS +RUN curl -fsSL https://pkg.labs.nic.cz/aptrepo/gpg.key \ + | gpg --dearmor -o /usr/share/keyrings/knot-labs.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/knot-labs.gpg] \ + https://pkg.labs.nic.cz/knot-dns bookworm main" \ + > /etc/apt/sources.list.d/knot-dns.list -RUN apt-get update && apt-get install -y \ - dnsutils \ +RUN apt-get update && apt-get install -y --no-install-recommends \ + knot \ + knot-dnsutils \ + iproute2 \ iptables \ - net-tools \ - dirmngr gnupg \ - # credentials management via envsubst - && apt-get -y install gettext-base \ - --no-install-recommends && apt-get clean && rm -rf /var/lib/apt/lists/* + gettext-base \ + && apt-get clean && rm -rf /var/lib/apt/lists/* -RUN set -ex \ - && echo 'deb [arch=amd64] http://repo.powerdns.com/ubuntu jammy-auth-48 main' \ - >> /etc/apt/sources.list \ - && echo 'Package: pdns-*' > /etc/apt/preferences.d/pdns \ - && echo 'Pin: origin repo.powerdns.com' >> /etc/apt/preferences.d/pdns \ - && echo 'Pin-Priority: 600' >> /etc/apt/preferences.d/pdns \ - && apt-key adv --keyserver keyserver.ubuntu.com --recv 0x1B0C6205FD380FBB +# Main config template (variables injected at startup via envsubst) +COPY conf/knot.conf.var /etc/knot/knot.conf.var -RUN apt-get update \ - && apt-get install -y pdns-server pdns-backend-pgsql postgresql-client-14 iproute2 \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && rm -rf /etc/powerdns/ +# Entrypoint and scripts +COPY entrypoint.sh /root/entrypoint.sh +COPY export-zones.sh /root/export-zones.sh +COPY import-zones.sh /root/import-zones.sh +RUN chmod +x /root/entrypoint.sh /root/export-zones.sh /root/import-zones.sh -COPY conf/ /etc/powerdns/ +CMD ["/root/entrypoint.sh"] diff --git a/nsmaster/conf/knot.conf.var b/nsmaster/conf/knot.conf.var new file mode 100644 index 000000000..501744bb4 --- /dev/null +++ b/nsmaster/conf/knot.conf.var @@ -0,0 +1,46 @@ +server: + rundir: "/run/knot" + user: knot:knot + listen: [ "0.0.0.0@53", "::@53" ] + # Persist runtime zone additions/removals in Knot's config database + config-db: /var/lib/knot/confdb + +log: + - target: stderr + any: info + +# ACL: allow external secondaries (connected via VPN) to AXFR from nsmaster +acl: + - id: acl-secondary-xfr + address: 10.8.0.0/24 + action: transfer + +remote: + # nslord is the zone master; its IP authorises both NOTIFYs and AXFR sources + - id: nslord + address: ${DESECSTACK_IPV4_REAR_PREFIX16}.1.11@53 + + # Multicast notify target inherited from the PowerDNS also-notify configuration + - id: notify-multicast + address: 239.1.2.3@53 + +# Template for zones slaved from nslord. +# catalog-zone causes each such zone to appear automatically in catalog.internal. +template: + - id: slave + master: nslord + acl: acl-secondary-xfr + notify: notify-multicast + catalog-zone: catalog.internal. # Knot maintains PTR entries here automatically + zonefile-sync: -1 # keep data in journal; no zone-file writes at runtime + zonefile-load: whole # load from file on first start if a file is present + journal-content: changes + +# Catalog zone: Knot generates its content from all zones with catalog-zone set above. +# Declared statically so it is present in the config-db from the very first startup. +zone: + - domain: catalog.internal. + catalog-role: generate + acl: acl-secondary-xfr + notify: notify-multicast + journal-content: changes diff --git a/nsmaster/conf/pdns.conf.var b/nsmaster/conf/pdns.conf.var deleted file mode 100644 index 6aa3803d3..000000000 --- a/nsmaster/conf/pdns.conf.var +++ /dev/null @@ -1,26 +0,0 @@ -api=yes -api-key=${DESECSTACK_NSMASTER_APIKEY} -allow-axfr-ips=10.8.0.0/24 -also-notify=239.1.2.3 -only-notify= -setgid=pdns -setuid=pdns -secondary=yes -secondary-do-renotify=yes -send-signed-notify=no -max-tcp-connections=200 -version-string=powerdns -webserver=yes -webserver-address=${DESECSTACK_IPV4_REAR_PREFIX16}.1.12 -webserver-allow-from=${DESECSTACK_IPV4_REAR_PREFIX16}.1.10 -webserver-max-bodysize=16 -carbon-server=${DESECSTACK_NSMASTER_CARBONSERVER} -carbon-ourname=${DESECSTACK_NSMASTER_CARBONOURNAME} - -launch=gpgsql -gpgsql-host=dbmaster -gpgsql-port=5432 -gpgsql-dbname=pdns -gpgsql-user=pdns -gpgsql-password=${DESECSTACK_DBMASTER_PASSWORD_pdns} -gpgsql-dnssec=yes diff --git a/nsmaster/entrypoint.sh b/nsmaster/entrypoint.sh index 48edb20ad..b765fc887 100755 --- a/nsmaster/entrypoint.sh +++ b/nsmaster/entrypoint.sh @@ -1,20 +1,25 @@ -#!/bin/bash +#!/usr/bin/env bash +set -euo pipefail -# Route required for communicating with secondaries through VPN -/sbin/ip route add 10.8.0.0/24 via 172.16.7.2 -/sbin/ip route add 239.1.2.0/24 via 172.16.7.2 +# VPN routing (same as before) +/sbin/ip route add 10.8.0.0/24 via ${DESECSTACK_IPV4_REAR_PREFIX16}.7.2 || true +/sbin/ip route add 239.1.2.0/24 via ${DESECSTACK_IPV4_REAR_PREFIX16}.7.2 || true -# Fix UDP TTL which sometimes is one, causing packets intended for VPN clients to be dropped at the VPN server -# TODO remove this workaround once the problem has been solved at its root -iptables -t mangle -A OUTPUT -p udp -j TTL --ttl-set 64 +# Fix UDP TTL (same workaround as before) +iptables -t mangle -A OUTPUT -p udp -j TTL --ttl-set 64 || true -# wait for dbmaster database to come up -until PGPASSWORD=$DESECSTACK_DBMASTER_PASSWORD_pdns psql -h dbmaster -U pdns -c '\q'; do - >&2 echo "Postgres is unavailable - sleeping" - sleep 1 -done +# Inject environment variables into Knot config +envsubst < /etc/knot/knot.conf.var > /etc/knot/knot.conf +chown knot:knot /etc/knot/knot.conf -# Manage credentials -envsubst < /etc/powerdns/pdns.conf.var > /etc/powerdns/pdns.conf +# Ensure storage and socket directories exist with correct ownership +mkdir -p /var/lib/knot /run/knot +chown -R knot:knot /var/lib/knot /run/knot +# 777 on the socket directory so the api container (different uid) can reach the socket. +# The socket itself is created world-accessible via umask 0 below. +chmod 777 /run/knot -exec pdns_server --daemon=no +# umask 0 is inherited by knotd, causing the control socket to be created with mode 0777, +# which allows the api container's process to connect without a shared GID. +umask 0 +exec knotd -c /etc/knot/knot.conf From 067a7c1c1ec87c15a22d8fe152ec4419f93e828b Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Tue, 26 May 2026 09:59:56 +0200 Subject: [PATCH 12/16] feat(docker-compose): remove dbmaster/rearmaster; wire up new Knot nsmaster (Steps 3+4d) Services: - Remove dbmaster service and its dbmaster_postgres volume reference; Knot stores zone data in its own journal (nsmaster_knot volume) - nsmaster: drop depends_on dbmaster; drop rearmaster network; drop env vars DESECSTACK_DBMASTER_PASSWORD_pdns / DESECSTACK_NSMASTER_APIKEY / DESECSTACK_NSMASTER_CARBONSERVER / DESECSTACK_NSMASTER_CARBONOURNAME; add volumes nsmaster_knot (journal/confdb), nsmaster_socket (control socket), zones-export bind-mount (read-only, used once at import time) - api: add nsmaster_socket volume mount at /run/knot; replace DESECSTACK_NSMASTER_APIKEY env var with DESECSTACK_NSMASTER_SOCKET - celery-email: remove DESECSTACK_NSMASTER_APIKEY (celery workers handle email only and do not interact with nsmaster) Networks: - Remove rearmaster network definition (172.16.4.0/24); nsmaster no longer connects to a Postgres backend Volumes: - Remove dbmaster_postgres - Add nsmaster_knot and nsmaster_socket .gitignore: - Add /zones-export/ to avoid accidentally committing exported zone files Co-Authored-By: Claude Sonnet 4.6 --- .gitignore | 4 ++++ docker-compose.yml | 50 +++++++++------------------------------------- 2 files changed, 13 insertions(+), 41 deletions(-) diff --git a/.gitignore b/.gitignore index ac6ff564f..8603244ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # python virtualenv and test files .env +.env.dev api/venv # IDE files @@ -11,6 +12,9 @@ api/venv # local certificates /certs/ +# Zone export directory (populated at migration time, not tracked) +/zones-export/ + # Webapp development files node_modules package-lock.json diff --git a/docker-compose.yml b/docker-compose.yml index e4883849e..b7e1e2111 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -97,30 +97,6 @@ services: tag: "desec/dblord" restart: unless-stopped - dbmaster: - build: dbmaster - image: desec/dedyn-dbmaster:latest - init: true - user: postgres:postgres - shm_size: 256M - volumes: - - dbmaster_postgres:/var/lib/postgresql/data - - ./dbmaster/pg_hba.conf:/usr/local/src/pg_hba.conf:ro - environment: - - POSTGRES_DB=pdns - - POSTGRES_HOST_AUTH_METHOD=reject - - POSTGRES_INITDB_ARGS=--auth-host=scram-sha-256 - - POSTGRES_NON_ROOT_USER=pdns - - POSTGRES_NON_ROOT_USER_PASSWORD=${DESECSTACK_DBMASTER_PASSWORD_pdns} - networks: - - rearmaster - command: ["postgres", "-c", "hba_file=/usr/local/src/pg_hba.conf"] - logging: - driver: "syslog" - options: - tag: "desec/dbmaster" - restart: unless-stopped - api: build: api image: desec/dedyn-api:latest @@ -133,6 +109,8 @@ services: - memcached tmpfs: - /var/local/django_metrics:size=500m + volumes: + - nsmaster_socket:/run/knot # rw: create/delete zones via knotc conf-set environment: - DESECSTACK_DOMAIN - DESECSTACK_NS @@ -155,7 +133,7 @@ services: - DESECSTACK_IPV6_SUBNET - DESECSTACK_NSLORD_APIKEY - DESECSTACK_NSLORD_DEFAULT_TTL - - DESECSTACK_NSMASTER_APIKEY + - DESECSTACK_NSMASTER_SOCKET=/run/knot/knot.sock - DESECSTACK_MINIMUM_TTL_DEFAULT - DESECSTACK_WATCHDOG_SECONDARIES networks: @@ -203,18 +181,15 @@ services: - NET_ADMIN ports: - "${DESECSTACK_PORT_XFR}:53" + volumes: + - nsmaster_knot:/var/lib/knot # Knot zone journal, config DB, zone files + - nsmaster_socket:/run/knot # control socket, shared with api container + - ./zones-export:/zones-export:ro # available for the one-shot import (Step 5) environment: - DESECSTACK_IPV4_REAR_PREFIX16 - - DESECSTACK_DBMASTER_PASSWORD_pdns - - DESECSTACK_NSMASTER_APIKEY - - DESECSTACK_NSMASTER_CARBONSERVER - - DESECSTACK_NSMASTER_CARBONOURNAME - depends_on: - - dbmaster networks: rearapi_ns: ipv4_address: ${DESECSTACK_IPV4_REAR_PREFIX16}.1.12 - rearmaster: rearreplication: ipv4_address: ${DESECSTACK_IPV4_REAR_PREFIX16}.7.3 logging: @@ -264,7 +239,6 @@ services: - DESECSTACK_IPV6_SUBNET - DESECSTACK_NSLORD_APIKEY - DESECSTACK_NSLORD_DEFAULT_TTL - - DESECSTACK_NSMASTER_APIKEY - DESECSTACK_MINIMUM_TTL_DEFAULT - DJANGO_SETTINGS_MODULE=api.settings networks: @@ -359,7 +333,8 @@ services: volumes: dbapi_postgres: dblord_mysql: - dbmaster_postgres: + nsmaster_knot: + nsmaster_socket: openvpn-server_logs: prometheus: rabbitmq_data: @@ -415,13 +390,6 @@ networks: config: - subnet: ${DESECSTACK_IPV4_REAR_PREFIX16}.3.0/24 gateway: ${DESECSTACK_IPV4_REAR_PREFIX16}.3.1 - rearmaster: - driver: bridge - ipam: - driver: default - config: - - subnet: ${DESECSTACK_IPV4_REAR_PREFIX16}.4.0/24 - gateway: ${DESECSTACK_IPV4_REAR_PREFIX16}.4.1 rearreplication: driver: bridge ipam: From b472ef4647d9b6ff4332ac249d967d13e00729d3 Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Tue, 26 May 2026 10:02:55 +0200 Subject: [PATCH 13/16] =?UTF-8?q?feat(api):=20replace=20PowerDNS=20nsmaste?= =?UTF-8?q?r=20calls=20with=20Knot=20DNS=20libknot=20(Step=206b=E2=80=936g?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit api/desecapi/knot.py (new): - Thin wrapper around libknot.control.KnotCtl for zone management on nsmaster - create_zone(name): conf-begin / conf-set zone + template / conf-commit - delete_zone(name): conf-begin / conf-unset zone / conf-commit - retrieve_zone(name): zone-retransfer (replaces axfr-retrieve HTTP PUT) - get_serials(): zone-status, returns {zone: serial} dict - SOCKET_PATH read from DESECSTACK_NSMASTER_SOCKET env var api/desecapi/pdns.py: - Remove NSMASTER sentinel and its _config entry - Remove create_zone_master, delete_zone_master (→ knot.py) - Remove axfr_to_master (→ knot.retrieve_zone) - Remove update_catalog, construct_catalog_rrset (Knot handles catalog) - Remove get_serials (→ knot.get_serials) - Remove unused gethostbyname_cached, sha1 import, socket import api/desecapi/pdns_change_tracker.py: - Import knot alongside pdns - CreateDomain.pdns_do: replace create_zone_master + update_catalog with knot.create_zone - DeleteDomain.pdns_do: replace delete_zone_master + update_catalog with knot.delete_zone - __exit__: replace pdns.axfr_to_master with knot.retrieve_zone api/desecapi/management/commands/check-secondaries.py: - Replace pdns.get_serials() with knot.get_serials() api/api/settings.py: - Remove NSMASTER_PDNS_API, NSMASTER_PDNS_API_TOKEN (no HTTP API on nsmaster) - Remove CATALOG_ZONE (catalog.internal maintained automatically by Knot) - Keep PDNS_MAX_BODY_SIZE (still referenced for nslord request size checks) api/requirements.txt: - Add libknot~=3.5.0 (matches Knot DNS version from CZ.NIC vendor repo) Co-Authored-By: Claude Sonnet 4.6 --- api/api/settings.py | 5 +- api/desecapi/knot.py | 95 +++++++++++++++++++ .../management/commands/check-secondaries.py | 4 +- api/desecapi/pdns.py | 79 --------------- api/desecapi/pdns_change_tracker.py | 12 +-- api/requirements.txt | 1 + 6 files changed, 105 insertions(+), 91 deletions(-) create mode 100644 api/desecapi/knot.py diff --git a/api/api/settings.py b/api/api/settings.py index 5508791a8..c7c9ea6ea 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -170,12 +170,9 @@ PSL_RESOLVER = os.environ.get("DESECSTACK_API_PSL_RESOLVER") LOCAL_PUBLIC_SUFFIXES = {"dedyn.%s" % os.environ["DESECSTACK_DOMAIN"]} -# PowerDNS-related +# PowerDNS-related (nslord only; nsmaster is now Knot DNS, accessed via libknot) NSLORD_PDNS_API = "http://nslord:8081/api/v1/servers/localhost" NSLORD_PDNS_API_TOKEN = os.environ["DESECSTACK_NSLORD_APIKEY"] -NSMASTER_PDNS_API = "http://nsmaster:8081/api/v1/servers/localhost" -NSMASTER_PDNS_API_TOKEN = os.environ["DESECSTACK_NSMASTER_APIKEY"] -CATALOG_ZONE = "catalog.internal" # Celery # see https://docs.celeryproject.org/en/stable/history/whatsnew-4.0.html#latentcall-django-admonition diff --git a/api/desecapi/knot.py b/api/desecapi/knot.py new file mode 100644 index 000000000..04ec453ca --- /dev/null +++ b/api/desecapi/knot.py @@ -0,0 +1,95 @@ +""" +Thin wrapper around libknot's control socket for managing zones on nsmaster (Knot DNS). + +Replaces the nsmaster-facing functions that previously used the PowerDNS HTTP API: + + Old (pdns.py, server=NSMASTER) New (knot.py) + ───────────────────────────────── ────────────────────────────────────────── + create_zone_master(name) create_zone(name) + delete_zone_master(name) delete_zone(name) + axfr_to_master(zone) retrieve_zone(name) + get_serials() get_serials() + update_catalog / construct_catalog (removed — Knot updates catalog automatically) +""" + +import os + +from libknot.control import KnotCtl, KnotCtlType + +SOCKET_PATH = os.environ.get("DESECSTACK_NSMASTER_SOCKET", "/run/knot/knot.sock") +SLAVE_TEMPLATE = "slave" # must match template id in knot.conf + + +def _ctl() -> KnotCtl: + """Return a connected KnotCtl instance (caller must close it).""" + ctl = KnotCtl() + ctl.connect(SOCKET_PATH) + return ctl + + +def create_zone(name: str) -> None: + """Register a new slave zone on nsmaster.""" + name = name.rstrip(".") + "." + ctl = _ctl() + try: + ctl.send_block(cmd="conf-begin") + ctl.receive_block() + ctl.send_block(cmd="conf-set", section="zone", identifier=name) + ctl.receive_block() + ctl.send_block( + cmd="conf-set", + section="zone", + identifier=name, + item="template", + data=SLAVE_TEMPLATE, + ) + ctl.receive_block() + ctl.send_block(cmd="conf-commit") + ctl.receive_block() + finally: + ctl.send(KnotCtlType.END) + ctl.close() + + +def delete_zone(name: str) -> None: + """Remove a slave zone from nsmaster.""" + name = name.rstrip(".") + "." + ctl = _ctl() + try: + ctl.send_block(cmd="conf-begin") + ctl.receive_block() + ctl.send_block(cmd="conf-unset", section="zone", identifier=name) + ctl.receive_block() + ctl.send_block(cmd="conf-commit") + ctl.receive_block() + finally: + ctl.send(KnotCtlType.END) + ctl.close() + + +def retrieve_zone(name: str) -> None: + """Trigger an AXFR/IXFR retrieval from nslord for the given zone.""" + name = name.rstrip(".") + "." + ctl = _ctl() + try: + ctl.send_block(cmd="zone-retransfer", zone=name) + ctl.receive_block() + finally: + ctl.send(KnotCtlType.END) + ctl.close() + + +def get_serials() -> dict[str, int]: + """Return {zone_name: serial} for all zones known to nsmaster.""" + ctl = _ctl() + try: + ctl.send_block(cmd="zone-status") + data = ctl.receive_block() + return { + zone: int(info.get("serial", 0)) + for zone, info in data.items() + if info.get("serial") + } + finally: + ctl.send(KnotCtlType.END) + ctl.close() diff --git a/api/desecapi/management/commands/check-secondaries.py b/api/desecapi/management/commands/check-secondaries.py index ab2868987..ac1f5b885 100644 --- a/api/desecapi/management/commands/check-secondaries.py +++ b/api/desecapi/management/commands/check-secondaries.py @@ -8,7 +8,7 @@ from django.utils import timezone import dns.exception, dns.message, dns.query, dns.rdatatype -from desecapi import pdns +from desecapi import knot from desecapi.models import Domain @@ -76,7 +76,7 @@ def handle(self, *args, **options): ).values_list("name", flat=True) serials = { zone: s - for zone, s in pdns.get_serials().items() + for zone, s in knot.get_serials().items() if zone.rstrip(".") in recent_domain_names } diff --git a/api/desecapi/pdns.py b/api/desecapi/pdns.py index 4eaf58bf1..3f09c1bc6 100644 --- a/api/desecapi/pdns.py +++ b/api/desecapi/pdns.py @@ -1,8 +1,5 @@ import json import re -import socket -from functools import cache -from hashlib import sha1 import requests from django.conf import settings @@ -70,25 +67,15 @@ } NSLORD = object() -NSMASTER = object() _config = { NSLORD: { "base_url": settings.NSLORD_PDNS_API, "apikey": settings.NSLORD_PDNS_API_TOKEN, }, - NSMASTER: { - "base_url": settings.NSMASTER_PDNS_API, - "apikey": settings.NSMASTER_PDNS_API_TOKEN, - }, } -@cache -def gethostbyname_cached(host): - return socket.gethostbyname(host) - - def _pdns_request( method, *, server, path, data=None, accept="application/json", **kwargs ): @@ -198,19 +185,6 @@ def get_rrset_datas(domain): ] -def update_catalog(zone, delete=False): - """ - Updates the catalog zone information (`settings.CATALOG_ZONE`) for the given zone. - """ - content = _pdns_patch( - NSMASTER, - "/zones/" + pdns_id(settings.CATALOG_ZONE), - {"rrsets": [construct_catalog_rrset(zone=zone, delete=delete)]}, - ) - metrics.get("desecapi_pdns_catalog_updated").inc() - return content - - def create_zone_lord(name): name = name.rstrip(".") + "." _pdns_post( @@ -244,19 +218,6 @@ def create_zone_lord(name): ) -def create_zone_master(name): - name = name.rstrip(".") + "." - _pdns_post( - NSMASTER, - "/zones?rrsets=false", - { - "name": name, - "kind": "SLAVE", - "masters": [gethostbyname_cached("nslord")], - }, - ) - - def delete_zone(name, server): _pdns_delete(server, "/zones/" + pdns_id(name)) @@ -265,45 +226,5 @@ def delete_zone_lord(name): _pdns_delete(NSLORD, "/zones/" + pdns_id(name)) -def delete_zone_master(name): - _pdns_delete(NSMASTER, "/zones/" + pdns_id(name)) - - def update_zone(name, data): _pdns_patch(NSLORD, "/zones/" + pdns_id(name), data) - - -def axfr_to_master(zone): - _pdns_put(NSMASTER, "/zones/%s/axfr-retrieve" % pdns_id(zone)) - - -def construct_catalog_rrset( - zone=None, delete=False, subname=None, qtype="PTR", rdata=None -): - # subname can be generated from zone for convenience; exactly one needs to be given - assert (zone is None) ^ (subname is None) - # sanity check: one can't delete an rrset and give record data at the same time - assert not (delete and rdata) - - if subname is None: - zone = zone.rstrip(".") + "." - m_unique = sha1(zone.encode()).hexdigest() - subname = f"{m_unique}.zones" - - if rdata is None: - rdata = zone - - return { - "name": f"{subname}.{settings.CATALOG_ZONE}".strip(".") + ".", - "type": qtype, - "ttl": 0, # as per the specification - "changetype": "REPLACE", - "records": [] if delete else [{"content": rdata, "disabled": False}], - } - - -def get_serials(): - return { - zone["name"]: zone["edited_serial"] - for zone in _pdns_get(NSMASTER, "/zones").json() - } diff --git a/api/desecapi/pdns_change_tracker.py b/api/desecapi/pdns_change_tracker.py index e03821f0d..aa8c8a45e 100644 --- a/api/desecapi/pdns_change_tracker.py +++ b/api/desecapi/pdns_change_tracker.py @@ -3,7 +3,7 @@ from django.db.transaction import atomic from django.utils import timezone -from desecapi import pch, pdns +from desecapi import knot, pch, pdns from desecapi.models import RRset, RR, Domain @@ -69,8 +69,8 @@ def axfr_required(self): def pdns_do(self): pdns.create_zone_lord(self.domain_name) - pdns.create_zone_master(self.domain_name) - pdns.update_catalog(self.domain_name) + knot.create_zone(self.domain_name) + # catalog.internal updated automatically by Knot def api_do(self): rr_set = RRset( @@ -97,8 +97,8 @@ def axfr_required(self): def pdns_do(self): pdns.delete_zone_lord(self.domain_name) - pdns.delete_zone_master(self.domain_name) - pdns.update_catalog(self.domain_name, delete=True) + knot.delete_zone(self.domain_name) + # catalog.internal updated automatically by Knot def api_do(self): pass @@ -258,7 +258,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.transaction.__exit__(None, None, None) for name in axfr_required: - pdns.axfr_to_master(name) + knot.retrieve_zone(name) Domain.objects.filter(name__in=axfr_required).update(published=timezone.now()) def _compute_changes(self): diff --git a/api/requirements.txt b/api/requirements.txt index 16de56b07..921ff4978 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,4 +1,5 @@ captcha~=0.7.1 +libknot~=3.5.0 celery~=5.6.2 coverage~=7.14.0 cryptography~=46.0.5 From 990f3ceb639b1bdb1347c0fde5a4da9c2c0a1500 Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Tue, 26 May 2026 10:11:01 +0200 Subject: [PATCH 14/16] chore(api): remove stale env vars and update test mocks for Knot DNS (Step 6h) .env.dev: - Remove DESECSTACK_DBMASTER_PASSWORD_pdns (no Postgres backend) - Remove DESECSTACK_NSMASTER_APIKEY (no HTTP API on nsmaster) - Remove DESECSTACK_NSMASTER_CARBONSERVER / _CARBONOURNAME (Knot has no carbon metric integration) - Add DESECSTACK_NSMASTER_SOCKET=/run/knot/knot.sock Management commands: - Delete align-catalog-zone.py: entirely obsolete; catalog.internal is generated and maintained automatically by Knot DNS - sync-to-pdns.py: replace pdns._pdns_put(NSMASTER, axfr-retrieve) with knot.retrieve_zone(); remove the catalog alignment call_command (no longer needed) views/domains.py: - Change 'from desecapi.pdns import get_serials' to knot.get_serials Test infrastructure (base.py): - MockPDNSTestCase.setUp(): remove request_pdns_zone_create(ns='MASTER') and request_pdns_zone_axfr() from blanket HTTP mocks; replace with mock.patch of desecapi.knot.{create_zone,delete_zone,retrieve_zone, get_serials} so tests run without a live Knot socket; mocks available as self.mock_knot_{create,delete,retrieve}_zone and self.mock_knot_get_serials for assertions - Remove request_pdns_zone_axfr / request_pdns_update_catalog class methods (these represented NSMASTER HTTP calls that no longer exist) - assertZoneCreation: remove ns='MASTER' zone create expectation; nsmaster is notified via socket (mocked above) - assertZoneDeletion: same - requests_desec_domain_creation: remove ns='MASTER' create, catalog update, and axfr expectations - requests_desec_domain_deletion: same for delete + catalog + axfr - requests_desec_domain_creation_auto_delegation: remove axfr expectation - requests_desec_rr_sets_update: remove axfr expectation - DynDomainOwnerTestCase: remove request_pdns_zone_axfr override; update assertDynDNS12Update to drop axfr expectation test_pdns_change_tracker.py: - assertPdnsZoneUpdate: remove request_pdns_zone_axfr; only the nslord HTTP update is asserted via assertRequests; knot.retrieve_zone is mocked test_replication.py: - Rewrite test_serials to configure self.mock_knot_get_serials instead of mocking an HTTP GET to nsmaster test_dyndns12update.py: - Remove all 12 self.request_pdns_zone_axfr() calls from assertRequests lists (these were checking the now-gone NSMASTER HTTP AXFR trigger) Co-Authored-By: Claude Sonnet 4.6 --- .env.dev | 10 +-- .../management/commands/align-catalog-zone.py | 72 ------------------- .../management/commands/sync-to-pdns.py | 31 +++----- api/desecapi/tests/base.py | 72 ++++++++----------- api/desecapi/tests/test_dyndns12update.py | 12 ---- .../tests/test_pdns_change_tracker.py | 3 +- api/desecapi/tests/test_replication.py | 26 ++----- api/desecapi/views/domains.py | 2 +- 8 files changed, 52 insertions(+), 176 deletions(-) delete mode 100644 api/desecapi/management/commands/align-catalog-zone.py diff --git a/.env.dev b/.env.dev index fb5100afe..39eb4bf72 100644 --- a/.env.dev +++ b/.env.dev @@ -26,6 +26,9 @@ DESECSTACK_API_PCH_API=https://localhost/pch/api DESECSTACK_API_PCH_API_TOKEN=insecure DESECSTACK_DBAPI_PASSWORD_desec=insecure DESECSTACK_MINIMUM_TTL_DEFAULT=3600 +# test +DESECSTACK_DJANGO_TEST=1 +DJANGO_SETTINGS_MODULE=api.settings_quick_test # nslord-related DESECSTACK_DBLORD_PASSWORD_pdns=insecure @@ -34,11 +37,8 @@ DESECSTACK_NSLORD_CARBONSERVER= DESECSTACK_NSLORD_CARBONOURNAME= DESECSTACK_NSLORD_DEFAULT_TTL=3600 -# nsmaster-related -DESECSTACK_DBMASTER_PASSWORD_pdns=insecure -DESECSTACK_NSMASTER_APIKEY=insecure -DESECSTACK_NSMASTER_CARBONSERVER= -DESECSTACK_NSMASTER_CARBONOURNAME= +# nsmaster-related (Knot DNS; no HTTP API, no Postgres backend) +DESECSTACK_NSMASTER_SOCKET=/run/knot/knot.sock # monitoring DESECSTACK_WATCHDOG_SECONDARIES= diff --git a/api/desecapi/management/commands/align-catalog-zone.py b/api/desecapi/management/commands/align-catalog-zone.py deleted file mode 100644 index 9b092a499..000000000 --- a/api/desecapi/management/commands/align-catalog-zone.py +++ /dev/null @@ -1,72 +0,0 @@ -from django.conf import settings -from django.core.management import BaseCommand - -from desecapi.exceptions import PDNSException -from desecapi.pdns import ( - _pdns_delete, - _pdns_get, - _pdns_post, - NSLORD, - NSMASTER, - pdns_id, - construct_catalog_rrset, -) - - -class Command(BaseCommand): - # https://tools.ietf.org/html/draft-muks-dnsop-dns-catalog-zones-04 - help = "Generate a catalog zone on nsmaster, based on zones known on nslord." - - def add_arguments(self, parser): - pass - - def handle(self, *args, **options): - catalog_zone_id = pdns_id(settings.CATALOG_ZONE) - - # Fetch zones from NSLORD - response = _pdns_get(NSLORD, "/zones").json() - zones = {zone["name"] for zone in response} - - # Retrieve catalog zone serial (later reused for recreating the catalog zone, for allow for smooth rollover) - try: - response = _pdns_get(NSMASTER, f"/zones/{catalog_zone_id}") - serial = response.json()["serial"] - except PDNSException as e: - if e.response.status_code == 404: - serial = None - else: - raise e - - # Purge catalog zone if exists - try: - _pdns_delete(NSMASTER, f"/zones/{catalog_zone_id}") - except PDNSException as e: - if e.response.status_code != 404: - raise e - - # Create new catalog zone - rrsets = [ - construct_catalog_rrset( - subname="", qtype="NS", rdata="invalid." - ), # as per the specification - construct_catalog_rrset( - subname="version", qtype="TXT", rdata='"2"' - ), # as per the specification - *(construct_catalog_rrset(zone=zone) for zone in zones), - ] - - data = { - "name": settings.CATALOG_ZONE + ".", - "kind": "MASTER", - "dnssec": False, # as per the specification - "nameservers": [], - "rrsets": rrsets, - } - - if serial is not None: - data["serial"] = ( - serial + 1 - ) # actually, pdns does increase this as well, but let's not rely on this - - _pdns_post(NSMASTER, "/zones?rrsets=false", data) - print(f"Aligned catalog zone ({len(zones)} member zones).") diff --git a/api/desecapi/management/commands/sync-to-pdns.py b/api/desecapi/management/commands/sync-to-pdns.py index 66c1d89ac..ef2518815 100644 --- a/api/desecapi/management/commands/sync-to-pdns.py +++ b/api/desecapi/management/commands/sync-to-pdns.py @@ -1,10 +1,10 @@ -from django.core.management import BaseCommand, CommandError, call_command -from django.db import transaction +from django.core.management import BaseCommand, CommandError -from desecapi import pdns +from desecapi import knot, pdns from desecapi.exceptions import PDNSException from desecapi.models import Domain from desecapi.pdns_change_tracker import PDNSChangeTracker +from django.db import transaction class Command(BaseCommand): @@ -28,39 +28,28 @@ def handle(self, *args, **options): if domain_name not in domain_names: raise CommandError("{} is not a known domain".format(domain_name)) - catalog_alignment = False for domain in domains: self.stdout.write("%s ..." % domain.name, ending="") try: - created = self._sync_domain(domain) - if created: - self.stdout.write(f" created (was missing) ...", ending="") - catalog_alignment = True + self._sync_domain(domain) self.stdout.write(" synced") except Exception as e: self.stdout.write(" failed") msg = "Error while processing {}: {}".format(domain.name, e) raise CommandError(msg) - if catalog_alignment: - call_command("align-catalog-zone") - @staticmethod @transaction.atomic def _sync_domain(domain): - created = False - - # Create domain on pdns if it does not exist + # Create domain on pdns/knot if it does not exist try: PDNSChangeTracker.CreateDomain(domain_name=domain.name).pdns_do() except PDNSException as e: - # Domain already exists + # Domain already exists on nslord if e.response.status_code == 409: pass else: raise e - else: - created = True # modifications actually merged with additions in CreateUpdateDeleteRRSets modifications = { @@ -71,12 +60,8 @@ def _sync_domain(domain): } - modifications deletions.discard(("SOA", "")) # do not remove SOA record - # Update zone on nslord, propagate to nsmaster + # Update zone on nslord, then trigger retrieval on nsmaster (Knot) PDNSChangeTracker.CreateUpdateDeleteRRSets( domain.name, set(), modifications, deletions ).pdns_do() - pdns._pdns_put( - pdns.NSMASTER, "/zones/{}/axfr-retrieve".format(pdns.pdns_id(domain.name)) - ) - - return created + knot.retrieve_zone(domain.name) diff --git a/api/desecapi/tests/base.py b/api/desecapi/tests/base.py index 39fa60753..440c30f2f 100644 --- a/api/desecapi/tests/base.py +++ b/api/desecapi/tests/base.py @@ -512,29 +512,11 @@ def request_pdns_zone_retrieve_crypto_keys(cls, name=None): "body": json.dumps(cls.get_body_pdns_zone_retrieve_crypto_keys()), } - @classmethod - def request_pdns_zone_axfr(cls, name=None): - return { - "method": "PUT", - "url": cls.get_full_pdns_url( - cls.PDNS_ZONE_AXFR, ns="MASTER", id=cls._pdns_zone_id_heuristic(name) - ), - "status": 200, - "body": "", - } - - @classmethod - def request_pdns_update_catalog(cls): - return { - "method": "PATCH", - "url": cls.get_full_pdns_url( - cls.PDNS_ZONE, - ns="MASTER", - id=cls._pdns_zone_id_heuristic("catalog.internal"), - ), - "status": 204, - "body": "", - } + # Note: request_pdns_zone_axfr and request_pdns_update_catalog have been removed. + # After the nsmaster migration to Knot DNS, zone retrieval (axfr-retrieve) and + # catalog updates are no longer HTTP calls to nsmaster. They are handled via: + # - knot.retrieve_zone() — socket call, mocked in MockPDNSTestCase.setUp() + # - catalog.internal — managed automatically by Knot, no API involvement def request_pch_zone_create(self, name): def request_callback(request): @@ -620,27 +602,27 @@ def assertNoRequestsBut(self, *expected_requests): def assertZoneCreation(self, name): """ - Asserts that nslord, nsmaster and PCH are contacted for zone creation. + Asserts that nslord and PCH are contacted for zone creation. + nsmaster (Knot DNS) is notified via socket (knot.create_zone), mocked separately. Name is only asserted for requests to PCH. """ return AssertRequestsContextManager( test_case=self, expected_requests=[ self.request_pdns_zone_create(ns="LORD"), - self.request_pdns_zone_create(ns="MASTER"), self.request_pch_zone_create(name=name), ], ) def assertZoneDeletion(self, name): """ - Asserts that nslord, nsmaster and PCH are contacted for zone deletion. + Asserts that nslord and PCH are contacted for zone deletion. + nsmaster (Knot DNS) is notified via socket (knot.delete_zone), mocked separately. """ return AssertRequestsContextManager( test_case=self, expected_requests=[ self.request_pdns_zone_delete(ns="LORD", name=name), - self.request_pdns_zone_delete(ns="MASTER", name=name), self.request_pch_zone_delete(name=name), ], ) @@ -702,8 +684,7 @@ def setUp(self): for request in [ # TODO delete not in this list - is this even needed? self.request_pdns_zone_create(ns="LORD"), - self.request_pdns_zone_create(ns="MASTER"), - self.request_pdns_zone_axfr(), + # nsmaster (Knot DNS): no HTTP mock needed; socket calls are patched below self.request_pdns_zone_update(), self.request_pdns_zone_retrieve_crypto_keys(), self.request_pdns_zone_retrieve(), @@ -713,6 +694,16 @@ def setUp(self): else: self.responses.add(**request) + # Patch all knot socket functions so tests run without a live Knot socket. + # Individual tests that want to assert these calls can inspect these mocks. + self.mock_knot_create_zone = mock.patch("desecapi.knot.create_zone").start() + self.mock_knot_delete_zone = mock.patch("desecapi.knot.delete_zone").start() + self.mock_knot_retrieve_zone = mock.patch("desecapi.knot.retrieve_zone").start() + self.mock_knot_get_serials = mock.patch( + "desecapi.knot.get_serials", return_value={} + ).start() + self.addCleanup(mock.patch.stopall) + class DesecTestCase(MockPDNSTestCase): """ @@ -907,12 +898,11 @@ def requests_desec_domain_creation(self, name=None, axfr=True, keys=True): soa_content = "get.desec.io. get.desec.io. 1 86400 3600 2419200 3600" requests = [ self.request_pdns_zone_create("LORD", body_matcher(soa_content)), - self.request_pdns_zone_create(ns="MASTER"), - self.request_pdns_update_catalog(), + # nsmaster zone registration: knot.create_zone() via socket, not HTTP + # catalog.internal: updated automatically by Knot, no HTTP mock needed self.request_pch_zone_create(name=name), ] - if axfr: - requests.append(self.request_pdns_zone_axfr(name=name)) + # axfr=True: knot.retrieve_zone() is called but mocked; no HTTP request if keys: requests.append(self.request_pdns_zone_retrieve_crypto_keys(name=name)) return requests @@ -920,8 +910,8 @@ def requests_desec_domain_creation(self, name=None, axfr=True, keys=True): def requests_desec_domain_deletion(self, domain): requests = [ self.request_pdns_zone_delete(name=domain.name, ns="LORD"), - self.request_pdns_zone_delete(name=domain.name, ns="MASTER"), - self.request_pdns_update_catalog(), + # nsmaster zone removal: knot.delete_zone() via socket, not HTTP + # catalog.internal: updated automatically by Knot, no HTTP mock needed self.request_pch_zone_delete(name=domain.name), ] @@ -929,7 +919,7 @@ def requests_desec_domain_deletion(self, domain): delegate_at = self._find_auto_delegation_zone(domain.name) requests += [ self.request_pdns_zone_update(name=delegate_at), - self.request_pdns_zone_axfr(name=delegate_at), + # delegation-zone retrieve: knot.retrieve_zone() via socket, not HTTP ] return requests @@ -938,14 +928,14 @@ def requests_desec_domain_creation_auto_delegation(self, name=None): delegate_at = self._find_auto_delegation_zone(name) return self.requests_desec_domain_creation(name=name) + [ self.request_pdns_zone_update(name=delegate_at), - self.request_pdns_zone_axfr(name=delegate_at), + # delegation-zone retrieve: knot.retrieve_zone() via socket, not HTTP ] @classmethod def requests_desec_rr_sets_update(cls, name=None): return [ cls.request_pdns_zone_update(name=name), - cls.request_pdns_zone_axfr(name=name), + # zone retrieve: knot.retrieve_zone() via socket, not HTTP ] def assertBadRequest(self, response, message, path=()): @@ -1222,10 +1212,6 @@ def setUp(self): class DynDomainOwnerTestCase(DomainOwnerTestCase): DYN = True - @classmethod - def request_pdns_zone_axfr(cls, name=None): - return super().request_pdns_zone_axfr(name.lower() if name else None) - @classmethod def request_pdns_zone_update(cls, name=None): return super().request_pdns_zone_update(name.lower() if name else None) @@ -1250,7 +1236,7 @@ def assertDynDNS12Update( ) requests = [ self.request_pdns_zone_update(name=pdns_name), - self.request_pdns_zone_axfr(name=pdns_name), + # zone retrieve: knot.retrieve_zone() via socket, not HTTP ] else: requests = [] diff --git a/api/desecapi/tests/test_dyndns12update.py b/api/desecapi/tests/test_dyndns12update.py index da426acaa..47456e72d 100644 --- a/api/desecapi/tests/test_dyndns12update.py +++ b/api/desecapi/tests/test_dyndns12update.py @@ -64,7 +64,6 @@ def test_deviant_ttl(self): """ with self.assertRequests( self.request_pdns_zone_update(self.my_domain.name), - self.request_pdns_zone_axfr(self.my_domain.name), ): response = self.client_token_authorized.patch_rr_set( self.my_domain.name.lower(), "", "A", {"ttl": 3600} @@ -80,7 +79,6 @@ def test_ddclient_dyndns1_v4_success(self): # /nic/dyndns?action=edit&started=1&hostname=YES&host_id=foobar.dedyn.io&myip=10.1.2.3 with self.assertRequests( self.request_pdns_zone_update(self.my_domain.name), - self.request_pdns_zone_axfr(self.my_domain.name), ): response = self.client.get( self.reverse("v1:dyndns12update"), @@ -327,7 +325,6 @@ def test_update_multiple_v4(self): with self.assertRequests( self.request_pdns_zone_update(domain1), - self.request_pdns_zone_axfr(domain1), ): response = self.client.get( self.reverse("v1:dyndns12update"), @@ -351,7 +348,6 @@ def test_update_multiple_with_overwrite(self): with self.assertRequests( self.request_pdns_zone_update(self.my_domain.name), - self.request_pdns_zone_axfr(self.my_domain.name), ): response = self.client.get( self.reverse("v1:dyndns12update"), @@ -383,7 +379,6 @@ def test_update_multiple_with_extra(self): with self.assertRequests( self.request_pdns_zone_update(self.my_domain.name), - self.request_pdns_zone_axfr(self.my_domain.name), ): response = self.client.get( self.reverse("v1:dyndns12update"), @@ -416,7 +411,6 @@ def test_update_multiple_with_same_argument_domain_twice(self): with self.assertRequests( self.request_pdns_zone_update(self.my_domain.name), - self.request_pdns_zone_axfr(self.my_domain.name), ): response = self.client.get( self.reverse("v1:dyndns12update"), @@ -443,7 +437,6 @@ def test_update_multiple_username_param(self): with self.assertRequests( self.request_pdns_zone_update(domain1), - self.request_pdns_zone_axfr(domain1), ): response = self.client_token_authorized.get( self.reverse("v1:dyndns12update"), @@ -465,7 +458,6 @@ def test_update_multiple_v4v6(self): with self.assertRequests( self.request_pdns_zone_update(domain1), - self.request_pdns_zone_axfr(domain1), ): response = self.client.get( self.reverse("v1:dyndns12update"), @@ -495,7 +487,6 @@ def test_update_multiple_with_subnet(self): with self.assertRequests( self.request_pdns_zone_update(self.my_domain.name), - self.request_pdns_zone_axfr(self.my_domain.name), ): response = self.client.get( self.reverse("v1:dyndns12update"), @@ -521,7 +512,6 @@ def test_update_multiple_with_subnet_and_ip_override(self): with self.assertRequests( self.request_pdns_zone_update(self.my_domain.name), - self.request_pdns_zone_axfr(self.my_domain.name), ): response = self.client.get( self.reverse("v1:dyndns12update"), @@ -547,7 +537,6 @@ def test_update_multiple_with_one_being_already_up_to_date(self): with self.assertRequests( self.request_pdns_zone_update(domain1), - self.request_pdns_zone_axfr(domain1), ): response = self.client.get( self.reverse("v1:dyndns12update"), @@ -566,7 +555,6 @@ def test_update_same_domain_twice(self): with self.assertRequests( self.request_pdns_zone_update(self.my_domain.name), - self.request_pdns_zone_axfr(self.my_domain.name), ): response = self.client.get( self.reverse("v1:dyndns12update"), diff --git a/api/desecapi/tests/test_pdns_change_tracker.py b/api/desecapi/tests/test_pdns_change_tracker.py index 1b63ceb38..6210ff6d5 100644 --- a/api/desecapi/tests/test_pdns_change_tracker.py +++ b/api/desecapi/tests/test_pdns_change_tracker.py @@ -23,10 +23,11 @@ def setUp(self): ) def assertPdnsZoneUpdate(self, name, rr_sets): + # After zone data changes, knot.retrieve_zone() is called (socket, not HTTP). + # The HTTP assertion here covers only the nslord zone update. return self.assertRequests( [ self.request_pdns_zone_update_assert_body(name, rr_sets), - self.request_pdns_zone_axfr(name), ], ) diff --git a/api/desecapi/tests/test_replication.py b/api/desecapi/tests/test_replication.py index 280cd28ad..43e94f92c 100644 --- a/api/desecapi/tests/test_replication.py +++ b/api/desecapi/tests/test_replication.py @@ -1,5 +1,3 @@ -import json - from rest_framework import status from desecapi.tests.base import DesecTestCase @@ -8,29 +6,19 @@ class ReplicationTest(DesecTestCase): def test_serials(self): url = self.reverse("v1:serial") - zones = [ - {"name": "test.example.", "edited_serial": 12345}, - {"name": "example.org.", "edited_serial": 54321}, - ] - serials = {zone["name"]: zone["edited_serial"] for zone in zones} - pdns_requests = [ - { - "method": "GET", - "url": self.get_full_pdns_url(r"/zones", ns="MASTER"), - "status": 200, - "body": json.dumps(zones), - } - ] + serials = {"test.example.": 12345, "example.org.": 54321} + + # knot.get_serials is patched by MockPDNSTestCase.setUp(); configure it here. + self.mock_knot_get_serials.return_value = serials # Run twice to make sure cache output varies on remote address for i in range(2): response = self.client.get(path=url, REMOTE_ADDR="123.8.0.2") self.assertStatus(response, status.HTTP_401_UNAUTHORIZED) - with self.assertRequests(pdns_requests): - response = self.client.get(path=url, REMOTE_ADDR="10.8.0.2") + response = self.client.get(path=url, REMOTE_ADDR="10.8.0.2") self.assertStatus(response, status.HTTP_200_OK) self.assertEqual(response.data, serials) - # Do not expect pdns request in next iteration (result will be cached) - pdns_requests = [] + # Do not expect knot call in next iteration (result will be cached) + self.mock_knot_get_serials.return_value = {} diff --git a/api/desecapi/views/domains.py b/api/desecapi/views/domains.py index 37ce115fc..80d4a3c9c 100644 --- a/api/desecapi/views/domains.py +++ b/api/desecapi/views/domains.py @@ -13,7 +13,7 @@ from desecapi import permissions from desecapi.models import Domain -from desecapi.pdns import get_serials +from desecapi.knot import get_serials from desecapi.pdns_change_tracker import PDNSChangeTracker from desecapi.renderers import PlainTextRenderer from desecapi.serializers import DomainSerializer From b3243b22634d3d6a817638cc0f3e17db4f154785 Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Tue, 26 May 2026 10:17:47 +0200 Subject: [PATCH 15/16] fixup: remove traces of old nsmaster/dbmaster --- .env.default | 8 -- .github/workflows/test.yml | 8 +- README.md | 11 +-- api/api/settings.py | 2 +- api/desecapi/exception_handlers.py | 2 +- dbmaster/Dockerfile | 14 --- .../init-user-db.sh | 16 --- .../schema.pgsql.sql.txt | 98 ------------------- dbmaster/entrypoint-wrapper.sh | 6 -- dbmaster/pg_hba.conf | 16 --- docker-compose.dev.yml | 4 - docker-compose.test-e2e2.yml | 1 - 12 files changed, 6 insertions(+), 180 deletions(-) delete mode 100644 dbmaster/Dockerfile delete mode 100644 dbmaster/docker-entrypoint-initdb.d/init-user-db.sh delete mode 100644 dbmaster/docker-entrypoint-initdb.d/schema.pgsql.sql.txt delete mode 100755 dbmaster/entrypoint-wrapper.sh delete mode 100644 dbmaster/pg_hba.conf diff --git a/.env.default b/.env.default index 84b9c7743..7b7b83557 100644 --- a/.env.default +++ b/.env.default @@ -34,14 +34,6 @@ DESECSTACK_NSLORD_CARBONSERVER= DESECSTACK_NSLORD_CARBONOURNAME= DESECSTACK_NSLORD_DEFAULT_TTL=3600 -# nsmaster-related -DESECSTACK_DBMASTER_PASSWORD_pdns= -DESECSTACK_NSMASTER_ALSO_NOTIFY= -DESECSTACK_NSMASTER_APIKEY= -DESECSTACK_NSMASTER_CARBONSERVER= -DESECSTACK_NSMASTER_CARBONOURNAME= -DESECSTACK_NSMASTER_TSIGKEY= - # monitoring DESECSTACK_WATCHDOG_SECONDARIES=ns1.example.org ns2.example.net DESECSTACK_PROMETHEUS_PASSWORD= diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fc1ceefab..5c6711462 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,10 +30,6 @@ env: DESECSTACK_DBAPI_PASSWORD_desec: 9Fn33T5yGueeee DESECSTACK_NSLORD_APIKEY: 9Fn33T5yGukjekwjew DESECSTACK_NSLORD_DEFAULT_TTL: 1234 - DESECSTACK_DBMASTER_PASSWORD_pdns: 9Fn33T5yGukjwelt - DESECSTACK_NSMASTER_ALSO_NOTIFY: - DESECSTACK_NSMASTER_APIKEY: LLq1orOQuXCINUz4TV - DESECSTACK_NSMASTER_TSIGKEY: +++undefined/undefined/undefined/undefined/undefined/undefined/undefined/undefined+++A== DESECSTACK_IPV4_REAR_PREFIX16: 172.16 DESECSTACK_IPV6_SUBNET: bade:affe:dead:beef:b011::/80 DESECSTACK_IPV6_ADDRESS: bade:affe:dead:beef:b011:0642:ac10:0080 @@ -62,7 +58,7 @@ jobs: - name: Build images uses: ./.github/workflows/build with: - images: api dbapi nslord nsmaster dblord dbmaster + images: api dbapi nslord nsmaster dblord - name: Check for missing migrations run: docker compose run -T api sh -c "./wait-dbapi && python manage.py makemigrations --check" @@ -92,7 +88,7 @@ jobs: - name: Build images uses: ./.github/workflows/build with: - images: api dbapi nslord nsmaster dblord dbmaster + images: api dbapi nslord nsmaster dblord - name: Run API Tests run: docker compose -f docker-compose.yml -f docker-compose.test-api.yml run -T api bash -c "./entrypoint-tests.sh" - name: API Tests Logs and Cleanup diff --git a/README.md b/README.md index 4a1321474..e442f90bd 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ deSEC Stack This is a docker compose application providing the basic stack for deSEC name services. It consists of - `nslord`: Eventually authoritative DNS server (PowerDNS). DNSSEC keying material is generated here. -- `nsmaster`: Stealth authoritative DNS server (PowerDNS). Receives fully signed AXFR zone transfers from `nslord`. No access to keys. +- `nsmaster`: Intermediary authoritative DNS server (Knot DNS). Receives fully signed AXFR zone transfers from `nslord`. No access to keys. - `api`: RESTful API to create deSEC users and domains, see [documentation](https://desec.readthedocs.io/). -- `dbapi`, `dblord`, `dbmaster`: Postgres databases for `api` and `nsmaster`, MariaDB database for `nslord`, respectively. +- `dbapi`, `dblord`: Postgres database for `api`, MariaDB database for `nslord`, respectively. - `www`: nginx instance serving static website content and proxying to `api` - `celery`: A shadow instance of the `api` code for performing asynchronous tasks (email delivery). - `rabbitmq`: `celery`'s queue @@ -61,13 +61,6 @@ Although most configuration is contained in this repository, some external depen - `DESECSTACK_NSLORD_CARBONSERVER`: pdns `carbon-server` setting on nslord (optional) - `DESECSTACK_NSLORD_CARBONOURNAME`: pdns `carbon-ourname` setting on nslord (optional) - `DESECSTACK_NSLORD_DEFAULT_TTL`: TTL to use by default, including for default NS records - - nsmaster-related - - `DESECSTACK_DBMASTER_PASSWORD_pdns`: mysql password for pdns on nsmaster - - `DESECSTACK_NSMASTER_ALSO_NOTIFY`: Comma-separated list of additional IP addresses to notify of zone updates - - `DESECSTACK_NSMASTER_APIKEY`: pdns API key on nsmaster (required so that we can execute zone deletions on nsmaster, which replicates to the secondaries) - - `DESECSTACK_NSMASTER_CARBONSERVER`: pdns `carbon-server` setting on nsmaster (optional) - - `DESECSTACK_NSMASTER_CARBONOURNAME`: pdns `carbon-ourname` setting on nsmaster (optional) - - `DESECSTACK_NSMASTER_TSIGKEY`: Base64-encoded value of the default TSIG key used for talking to external secondaries (algorithm: HMAC-SHA256) - monitoring-related - `DESECSTACK_WATCHDOG_SECONDARIES`: space-separated list of secondary hostnames; used to check correct replication of recent DNS changes - `DESECSTACK_PROMETHEUS_PASSWORD`: basic auth password for user `prometheus` at `https://${DESECSTACK_DOMAIN}/prometheus/` diff --git a/api/api/settings.py b/api/api/settings.py index c7c9ea6ea..75c557b85 100644 --- a/api/api/settings.py +++ b/api/api/settings.py @@ -170,7 +170,7 @@ PSL_RESOLVER = os.environ.get("DESECSTACK_API_PSL_RESOLVER") LOCAL_PUBLIC_SUFFIXES = {"dedyn.%s" % os.environ["DESECSTACK_DOMAIN"]} -# PowerDNS-related (nslord only; nsmaster is now Knot DNS, accessed via libknot) +# nslord NSLORD_PDNS_API = "http://nslord:8081/api/v1/servers/localhost" NSLORD_PDNS_API_TOKEN = os.environ["DESECSTACK_NSLORD_APIKEY"] diff --git a/api/desecapi/exception_handlers.py b/api/desecapi/exception_handlers.py index 0a1b477d4..11a675c1b 100644 --- a/api/desecapi/exception_handlers.py +++ b/api/desecapi/exception_handlers.py @@ -38,7 +38,7 @@ def _500(): handlers = { IntegrityError: _409, OSError: _500, # OSError happens on system-related errors, like full disk or getaddrinfo() failure. - PDNSException: _500, # nslord/nsmaster returned an error + PDNSException: _500, # nslord returned an error } for exception_class, handler in handlers.items(): diff --git a/dbmaster/Dockerfile b/dbmaster/Dockerfile deleted file mode 100644 index 0a0ea0e91..000000000 --- a/dbmaster/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM postgres:13-alpine - -RUN apk add --no-cache pwgen - -USER postgres - -# mountable storage -VOLUME /var/lib/postgresql/data - -ENTRYPOINT ["entrypoint-wrapper.sh"] -CMD ["postgres"] - -COPY entrypoint-wrapper.sh /usr/local/bin/ -ADD docker-entrypoint-initdb.d /docker-entrypoint-initdb.d diff --git a/dbmaster/docker-entrypoint-initdb.d/init-user-db.sh b/dbmaster/docker-entrypoint-initdb.d/init-user-db.sh deleted file mode 100644 index 1bcd6cb93..000000000 --- a/dbmaster/docker-entrypoint-initdb.d/init-user-db.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -set -e - -# Get the postgres user or set it to a default value -if [ -n $POSTGRES_USER ]; then pg_user=$POSTGRES_USER; else pg_user="postgres"; fi -# Get the postgres db or set it to a default value -if [ -n $POSTGRES_DB ]; then pg_db=$POSTGRES_DB; else pg_db=$POSTGRES_USER; fi - -if [ -n "$POSTGRES_NON_ROOT_USER" ]; then -psql -v ON_ERROR_STOP=1 --username "$pg_user" --dbname "$pg_db" <<-EOSQL - CREATE USER $POSTGRES_NON_ROOT_USER with encrypted password '$POSTGRES_NON_ROOT_USER_PASSWORD'; - GRANT CREATE, CONNECT ON DATABASE $pg_db TO $POSTGRES_NON_ROOT_USER; - ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, UPDATE, INSERT, DELETE, REFERENCES ON TABLES TO $POSTGRES_NON_ROOT_USER; -EOSQL -psql -v ON_ERROR_STOP=1 --username "$POSTGRES_NON_ROOT_USER" --dbname "$pg_db" < /docker-entrypoint-initdb.d/schema.pgsql.sql.txt -fi diff --git a/dbmaster/docker-entrypoint-initdb.d/schema.pgsql.sql.txt b/dbmaster/docker-entrypoint-initdb.d/schema.pgsql.sql.txt deleted file mode 100644 index 10f542c9f..000000000 --- a/dbmaster/docker-entrypoint-initdb.d/schema.pgsql.sql.txt +++ /dev/null @@ -1,98 +0,0 @@ -CREATE TABLE domains ( - id SERIAL PRIMARY KEY, - name VARCHAR(255) NOT NULL, - master VARCHAR(128) DEFAULT NULL, - last_check INT DEFAULT NULL, - type TEXT NOT NULL, - notified_serial BIGINT DEFAULT NULL, - account VARCHAR(40) DEFAULT NULL, - options TEXT DEFAULT NULL, - catalog TEXT DEFAULT NULL, - CONSTRAINT c_lowercase_name CHECK (((name)::TEXT = LOWER((name)::TEXT))) -); - -CREATE UNIQUE INDEX name_index ON domains(name); -CREATE INDEX catalog_idx ON domains(catalog); - - -CREATE TABLE records ( - id BIGSERIAL PRIMARY KEY, - domain_id INT DEFAULT NULL, - name VARCHAR(255) DEFAULT NULL, - type VARCHAR(10) DEFAULT NULL, - content VARCHAR(65535) DEFAULT NULL, - ttl INT DEFAULT NULL, - prio INT DEFAULT NULL, - disabled BOOL DEFAULT 'f', - ordername VARCHAR(255), - auth BOOL DEFAULT 't', - CONSTRAINT domain_exists - FOREIGN KEY(domain_id) REFERENCES domains(id) - ON DELETE CASCADE, - CONSTRAINT c_lowercase_name CHECK (((name)::TEXT = LOWER((name)::TEXT))) -); - -CREATE INDEX rec_name_index ON records(name); -CREATE INDEX nametype_index ON records(name,type); -CREATE INDEX domain_id ON records(domain_id); -CREATE INDEX recordorder ON records (domain_id, ordername text_pattern_ops); - - -CREATE TABLE supermasters ( - ip INET NOT NULL, - nameserver VARCHAR(255) NOT NULL, - account VARCHAR(40) NOT NULL, - PRIMARY KEY(ip, nameserver) -); - - -CREATE TABLE comments ( - id SERIAL PRIMARY KEY, - domain_id INT NOT NULL, - name VARCHAR(255) NOT NULL, - type VARCHAR(10) NOT NULL, - modified_at INT NOT NULL, - account VARCHAR(40) DEFAULT NULL, - comment VARCHAR(65535) NOT NULL, - CONSTRAINT domain_exists - FOREIGN KEY(domain_id) REFERENCES domains(id) - ON DELETE CASCADE, - CONSTRAINT c_lowercase_name CHECK (((name)::TEXT = LOWER((name)::TEXT))) -); - -CREATE INDEX comments_domain_id_idx ON comments (domain_id); -CREATE INDEX comments_name_type_idx ON comments (name, type); -CREATE INDEX comments_order_idx ON comments (domain_id, modified_at); - - -CREATE TABLE domainmetadata ( - id SERIAL PRIMARY KEY, - domain_id INT REFERENCES domains(id) ON DELETE CASCADE, - kind VARCHAR(32), - content TEXT -); - -CREATE INDEX domainidmetaindex ON domainmetadata(domain_id); - - -CREATE TABLE cryptokeys ( - id SERIAL PRIMARY KEY, - domain_id INT REFERENCES domains(id) ON DELETE CASCADE, - flags INT NOT NULL, - active BOOL, - published BOOL DEFAULT TRUE, - content TEXT -); - -CREATE INDEX domainidindex ON cryptokeys(domain_id); - - -CREATE TABLE tsigkeys ( - id SERIAL PRIMARY KEY, - name VARCHAR(255), - algorithm VARCHAR(50), - secret VARCHAR(255), - CONSTRAINT c_lowercase_name CHECK (((name)::TEXT = LOWER((name)::TEXT))) -); - -CREATE UNIQUE INDEX namealgoindex ON tsigkeys(name, algorithm); diff --git a/dbmaster/entrypoint-wrapper.sh b/dbmaster/entrypoint-wrapper.sh deleted file mode 100755 index 0bc605fc5..000000000 --- a/dbmaster/entrypoint-wrapper.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env bash -set -Eeo pipefail - -# This password is set for the postgres user when initializing the database. It is not needed and thus not printed. -export POSTGRES_PASSWORD=$(pwgen -1 -s 32) -/usr/local/bin/docker-entrypoint.sh "$@" diff --git a/dbmaster/pg_hba.conf b/dbmaster/pg_hba.conf deleted file mode 100644 index 76b931a20..000000000 --- a/dbmaster/pg_hba.conf +++ /dev/null @@ -1,16 +0,0 @@ -# TYPE DATABASE USER ADDRESS METHOD - -# "local" is for Unix domain socket connections only -local all all trust -# IPv4 local connections: -#host all all 127.0.0.1/32 scram-sha-256 -# IPv6 local connections: -#host all all ::1/128 scram-sha-256 -# Allow replication connections from localhost, by a user with the -# replication privilege. -#local replication all trust -#host replication all 127.0.0.1/32 scram-sha-256 -#host replication all ::1/128 scram-sha-256 - -host pdns pdns all scram-sha-256 -host all all all reject diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index d5b54e95b..38632e25e 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -22,10 +22,6 @@ services: logging: driver: "json-file" - dbmaster: - logging: - driver: "json-file" - api: environment: - DESECSTACK_API_DEBUG=True diff --git a/docker-compose.test-e2e2.yml b/docker-compose.test-e2e2.yml index 6496a483f..262c1977f 100644 --- a/docker-compose.test-e2e2.yml +++ b/docker-compose.test-e2e2.yml @@ -58,7 +58,6 @@ services: - DESECSTACK_IPV6_ADDRESS - DESECSTACK_NSLORD_DEFAULT_TTL - DESECSTACK_MINIMUM_TTL_DEFAULT - - DESECSTACK_NSMASTER_TSIGKEY - DESECSTACK_E2E2_SECONDARY_NS=${DESECSTACK_IPV4_REAR_PREFIX16}.0.130 # faketime setup - LD_PRELOAD=/lib/libfaketime.so From 9ca00f658098461ce4c744d32d1c66873294fcb5 Mon Sep 17 00:00:00 2001 From: Peter Thomassen Date: Tue, 26 May 2026 10:25:10 +0200 Subject: [PATCH 16/16] fixup: nsmaster build --- api/wait-ns | 2 +- nsmaster/Dockerfile | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/api/wait-ns b/api/wait-ns index 33b0cdbac..acc2a1d5f 100755 --- a/api/wait-ns +++ b/api/wait-ns @@ -3,4 +3,4 @@ set -e # wait for pdns api to come up host=nslord; port=8081; n=120; i=0; while ! (echo > /dev/tcp/$host/$port) 2> /dev/null; do [[ $i -eq $n ]] && >&2 echo "$host:$port not up after $n seconds, exiting" && exit 1; echo "waiting for $host:$port to come up"; sleep 1; i=$((i+1)); done -host=nsmaster; port=8081; n=120; i=0; while ! (echo > /dev/tcp/$host/$port) 2> /dev/null; do [[ $i -eq $n ]] && >&2 echo "$host:$port not up after $n seconds, exiting" && exit 1; echo "waiting for $host:$port to come up"; sleep 1; i=$((i+1)); done +host=nsmaster; port=53; n=120; i=0; while ! (echo > /dev/tcp/$host/$port) 2> /dev/null; do [[ $i -eq $n ]] && >&2 echo "$host:$port not up after $n seconds, exiting" && exit 1; echo "waiting for $host:$port to come up"; sleep 1; i=$((i+1)); done diff --git a/nsmaster/Dockerfile b/nsmaster/Dockerfile index 5e82c5a39..bbe038881 100644 --- a/nsmaster/Dockerfile +++ b/nsmaster/Dockerfile @@ -2,17 +2,18 @@ FROM debian:bookworm-slim # Install prerequisites for adding the vendor apt repo RUN apt-get update && apt-get install -y --no-install-recommends \ + apt-transport-https \ curl \ gnupg \ ca-certificates \ && apt-get clean && rm -rf /var/lib/apt/lists/* # Add CZ.NIC vendor repository for Knot DNS -RUN curl -fsSL https://pkg.labs.nic.cz/aptrepo/gpg.key \ - | gpg --dearmor -o /usr/share/keyrings/knot-labs.gpg \ - && echo "deb [signed-by=/usr/share/keyrings/knot-labs.gpg] \ +RUN curl -fsSL https://pkg.labs.nic.cz/gpg \ + > /usr/share/keyrings/cznic-labs-pkg.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/cznic-labs-pkg.gpg] \ https://pkg.labs.nic.cz/knot-dns bookworm main" \ - > /etc/apt/sources.list.d/knot-dns.list + > /etc/apt/sources.list.d/cznic-labs-knot-dns.list RUN apt-get update && apt-get install -y --no-install-recommends \ knot \