the_prg_server_configuration/nix-system-configs/modules/system/traefik.nix

{
  config,
  pkgs,
  lib,
  ...
}: let
  choose = paths: builtins.head (builtins.filter (p: builtins.pathExists p) paths);
  # Package the upstream cloudflare-ddns-updater from GitHub into the Nix store
  upstreamOwner = "K0p1-Git";
  upstreamRepo = "cloudflare-ddns-updater";
  upstreamRev = "e9906c3aa0b73c26e3473618ad7a69db853e669d";
  upstreamSrc = pkgs.fetchFromGitHub {
    owner = upstreamOwner;
    repo = upstreamRepo;
    rev = upstreamRev;
    sha256 = "0j1pv57hk9rhr1kxqqjxg71y0d2d1hi3b4yiq908x5kcv3abbina";
  };

  cloudflare-ddns-pkg = pkgs.stdenv.mkDerivation {
    pname = "cloudflare-ddns-updater";
    version = upstreamRev;
    src = upstreamSrc;
    # no build required for a script-only repo
    buildPhase = '': '';
    installPhase = ''
      mkdir -p $out/bin $out/lib/cloudflare-ddns
      cp -r $src/* $out/lib/cloudflare-ddns/ || true
      if [ -x "$out/lib/cloudflare-ddns/update.sh" ]; then
        cp "$out/lib/cloudflare-ddns/update.sh" $out/bin/cloudflare-ddns
        chmod +x $out/bin/cloudflare-ddns
      else
        # If upstream layout changes, keep the repo contents available under $out/lib
        true
      fi
    '';
    meta = with pkgs.lib; {
      description = "Cloudflare DDNS updater";
      license = licenses.mit;
    };
  };
in {
  options.local = {
    hostname = lib.mkOption {
      type = lib.types.str;
      default = "nixos-default";
      description = "System hostname";
    };
    username = lib.mkOption {
      type = lib.types.str;
      default = "user";
      description = "Primary user username";
    };
    userDescription = lib.mkOption {
      type = lib.types.str;
      default = "NixOS User";
      description = "Primary user description";
    };
    address = lib.mkOption {
      type = lib.types.str;
      default = "10.1.1.100";
      description = "Static IP address";
    };
  };

  imports = [
    (choose [./modules/desktop-manager/sway_greetd_homemanager.nix ../desktop-manager/sway_greetd_homemanager.nix])
    (choose [./modules/local/hostname_username.nix ../local/hostname_username.nix])
    (choose [./modules/local/networking_local.nix ../local/networking_local.nix])
    (choose [./modules/bootloader/seabios-assigned-iso-at-birth.nix ../bootloader/seabios-assigned-iso-at-birth.nix])
    (choose [./modules/lix-default.nix ../lix-default.nix])
    # Optionally: (choose [ ./modules/toolsets/remote_building.nix ../toolsets/remote_building.nix ])
  ];

  config = {
    local.hostname = "nixos-traefik";
    local.username = "traefikprg";
    local.userDescription = "NixOS PRG Traefik Service";
    local.address = "10.1.1.250";

    # Configure Anubis service
    services.anubis = {
      instances = {
        "" = {
          enable = true;
          settings = {
            # Bind to TCP instead of Unix socket for Docker-style integration
            BIND_NETWORK = "tcp";
            BIND = "127.0.0.1:8090";

            # Empty TARGET for redirect mode
            TARGET = " ";

            # Configure redirect domains - ALL domains that should be able to redirect back after challenge
            REDIRECT_DOMAINS = "prg-radio.org,anubis.prg-radio.org,git.prg-radio.org,lgbtq.prg-radio.org,wavelog.prg-radio.org,partdb.prg-radio.org,grafana.prg-radio.org";

            # Public URL for Anubis
            PUBLIC_URL = "https://anubis.prg-radio.org";

            # Cookie domain for proper scoping
            COOKIE_DOMAIN = ".prg-radio.org";

            # Difficulty level
            DIFFICULTY = 7;
            #ALGOTIHM = "slow";

            # Optional: serve robots.txt
            SERVE_ROBOTS_TXT = true;

            # Optional: webmaster email for error pages
            WEBMASTER_EMAIL = "dtu.prg@gmail.com";

            # Prevents infinite loop of doom:
            # See: https://github.com/TecharoHQ/anubis/issues/970
            JWT_RESTRICTION_HEADER = "CF-Connecting-IP";

            # Metrics on separate port
            METRICS_BIND_NETWORK = "tcp";
            METRICS_BIND = "127.0.0.1:8091";

            SLOG_LEVEL = "DEBUG";

            COOKIE_SECURE = true;
            COOKIE_SAME_SITE = "None"; # Only if Secure=true and you need cross-site
          };
        };
      };
    };

    services.loki = {
      enable = true;
      configuration = {
        auth_enabled = false;

        server = {
          http_listen_port = 3100;
        };

        ingester = {
          lifecycler = {
            address = "0.0.0.0";
            ring = {
              kvstore = {
                store = "inmemory";
              };
              replication_factor = 1;
            };
            final_sleep = "0s";
          };
          chunk_idle_period = "1h";
          max_chunk_age = "1h";
          chunk_target_size = 1048576;
          chunk_retain_period = "30s";
        };

        schema_config = {
          configs = [
            {
              from = "2020-10-24";
              store = "boltdb-shipper";
              object_store = "filesystem";
              schema = "v11";
              index = {
                prefix = "index_";
                period = "24h";
              };
            }
          ];
        };

        storage_config = {
          boltdb_shipper = {
            active_index_directory = "/var/lib/loki/boltdb-shipper-active";
            cache_location = "/var/lib/loki/boltdb-shipper-cache";
            cache_ttl = "24h";
          };
          filesystem = {
            directory = "/var/lib/loki/chunks";
          };
        };

        limits_config = {
          reject_old_samples = true;
          reject_old_samples_max_age = "168h";
          # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used.
          allow_structured_metadata = false;
        };

        table_manager = {
          retention_deletes_enabled = false;
          retention_period = "0s";
        };
      };
    };

    services.alloy = {
      enable = true;
    };

    services.prometheus = {
      enable = true;
      port = 9001;

      exporters.node = {
        enabledCollectors = [
          "ethtool"
          "softirqs"
          "systemd"
          "tcpstat"
        ];
        enable = true;
        port = 9002;
      };
      globalConfig.scrape_interval = "5s"; # "1m"
      scrapeConfigs = [
        {
          job_name = "traefik";
          static_configs = [
            {
              targets = ["localhost:8080"];
            }
          ];
        }
      ];
    };

    # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md
    environment.etc."alloy/config.alloy" = {
      text = ''
        discovery.relabel "metrics_integrations_integrations_traefik" {
        	targets = [{
        		__address__ = "localhost:8080",
        	}]

        	rule {
        		target_label = "instance"
        		replacement  = constants.hostname
        	}
        }

        prometheus.scrape "metrics_integrations_integrations_traefik" {
        	targets    = discovery.relabel.metrics_integrations_integrations_traefik.output
        	job_name   = "integrations/traefik"
        }
      '';
      group = "root";
      mode = "0644";
    };

    services.traefik = {
      enable = true;
      group = "acme";
      staticConfigOptions = {
        metrics = {
          prometheus = {
            addEntryPointsLabels = true;
            addRoutersLabels = true;
            addServicesLabels = true;
            buckets = [0.1 0.3 1.2 5.0];
          };
        };

        entryPoints = {
          web = {
            # Bind on IPv6 as well as IPv4 by using [::]:80
            address = "[::]:80";
            asDefault = true;
            http.redirections.entrypoint = {
              to = "websecure";
              scheme = "https";
            };
          };
          websecure = {
            # Bind on IPv6 as well as IPv4 by using [::]:443
            address = "[::]:443";
            asDefault = true;
            http.tls = {
              domains = [
                {
                  main = "prg-radio.org";
                  sans = ["*.prg-radio.org"];
                }
              ];
            };
          };
          # Federation entrypoint: external TLS on 8448
          federation = {
            # Bind on IPv6 as well as IPv4 by using [::]:8448
            address = "[::]:8448";
          };
          # TeamSpeak entry points
          teamspeak-voice = {
            # UDP entrypoint bind to IPv6 as well
            address = "[::]:9987/udp";
          };
          teamspeak-squery = {
            address = "[::]:10022/tcp";
          };
          teamspeak-data = {
            address = "[::]:30033/tcp";
          };

          # Mail entrypoints: plain SMTP, implicit SMTPS (465) and IMAPS (993)
          smtp = {
            # plain SMTP (port 25) - TCP passthrough to backend
            address = "[::]:25";
          };
          smtps = {
            # implicit TLS SMTP (port 465) - passthrough to backend
            address = "[::]:465";
          };
          imaps = {
            # implicit TLS IMAP (port 993) - passthrough to backend
            address = "[::]:993";
          };
        };
        log = {
          level = "INFO";
          filePath = "${config.services.traefik.dataDir}/traefik.log";
          format = "json";
        };
        api.dashboard = true;
        api.insecure = true;
      };
      dynamicConfigOptions = {
        tls.certificates = [
          {
            certFile = "/var/lib/acme/prg-radio.org/cert.pem";
            keyFile = "/var/lib/acme/prg-radio.org/key.pem";
          }
        ];

        # Traefik middlewares: Anubis forwardAuth
        http.middlewares = {
          anubisForwardAuth = {
            forwardAuth = {
              # point at the Anubis check endpoint (the subrequest/UI instance)
              address = "http://127.0.0.1:8090/.within.website/x/cmd/anubis/api/check";
            };
          };
          redirect-to-https = {
            redirectScheme = {
              scheme = "https";
            };
          };
          # Middleware to strip the /management prefix when proxying to the mail admin
          mail-strip-management = {
            stripPrefix = {prefixes = ["/management"];};
          };
        };

        http.routers = {
          # Anubis router (for challenge page)
          anubis = {
            rule = "Host(`anubis.prg-radio.org`)";
            service = "anubis";
            entryPoints = ["websecure"];
            tls = {
              certresolver = "acme";
            };
          };

          # Protected service example: Forgejo (attach Anubis forwardAuth middleware)
          forgejo = {
            rule = "Host(`git.prg-radio.org`)";
            service = "forgejo";
            entryPoints = ["websecure"];
            tls = {};
            middlewares = ["anubisForwardAuth"];
          };

          # Matrix HTTP router for client requests (Element etc.)
          matrix = {
            rule = "Host(`lgbtq.prg-radio.org`)";
            service = "matrix";
            entryPoints = ["websecure"];
            tls = {};
          };

          # Protected service: Wavelog
          wavelog = {
            rule = "Host(`wavelog.prg-radio.org`)";
            service = "wavelog";
            entryPoints = ["websecure"];
            tls = {};
          };

          # Protected service: PartDB
          partdb = {
            rule = "Host(`partdb.prg-radio.org`)";
            service = "partdb";
            entryPoints = ["websecure"];
            tls = {};
            middlewares = ["anubisForwardAuth"];
          };

          # Protected service: Grafana
          grafana = {
            rule = "Host(`grafana.prg-radio.org`)";
            service = "grafana";
            entryPoints = ["websecure"];
            tls = {};
            middlewares = ["anubisForwardAuth"];
          };

          # Mail HTTP (JMAP / web) - terminate TLS at Traefik and forward to Stalwart JMAP HTTP listener
          mail = {
            rule = "Host(`mail.prg-radio.org`)";
            service = "mail-jmap";
            entryPoints = ["websecure"];
            tls = {certresolver = "acme";};
          };

          # Mail web administration UI (Stalwart management) - exposed under /management
          mail-webadmin = {
            rule = "Host(`mailadmin.prg-radio.org`) && PathPrefix(`/management`)";
            service = "mail-webadmin";
            entryPoints = ["websecure"];
            tls = {certresolver = "acme";};
            # strip the /management prefix before proxying to the backend
            middlewares = ["mail-strip-management"];
          };
        };

        http.services = {
          # Anubis service (challenge UI / redirect endpoint)
          anubis.loadBalancer = {
            servers = [
              {url = "http://127.0.0.1:8090";}
            ];
          };

          forgejo.loadBalancer = {
            servers = [
              {url = "http://10.1.1.4:30000";}
            ];
          };
          matrix.loadBalancer = {
            servers = [
              {url = "http://10.1.1.248:12244";}
            ];
          };
          wavelog.loadBalancer = {
            servers = [
              {url = "http://10.1.1.249:8086";}
            ];
          };

          partdb.loadBalancer = {
            servers = [
              {url = "http://10.1.1.249:8087";}
            ];
          };

          grafana.loadBalancer = {
            servers = [
              {url = "http://10.1.1.10:3000";}
            ];
          };

          # Mail JMAP HTTP backend
          mail-jmap.loadBalancer = {
            servers = [{url = "http://10.1.1.15:8080";}];
          };

          # Mail webadmin backend (management). Traefik will strip the /management
          # prefix and forward the remainder to the management listener on the mail
          # server which is bound to port 8081.
          mail-webadmin.loadBalancer = {
            servers = [{url = "http://10.1.1.15:8081";}];
          };
        };

        # TCP routing for TeamSpeak
        tcp.routers = {
          teamspeak-squery = {
            rule = "HostSNI(`*`)";
            service = "teamspeak-squery";
            entryPoints = ["teamspeak-squery"];
          };
          teamspeak-data = {
            rule = "HostSNI(`*`)";
            service = "teamspeak-data";
            entryPoints = ["teamspeak-data"];
          };
          # Federation TCP router: incoming on :8448 -> Conduit backend
          matrix-federation = {
            rule = "HostSNI(`*`)";
            service = "matrix-federation";
            entryPoints = ["federation"];
            # Pass TLS through to the backend (Conduit handles TLS on port 12244)
            tls = {
              passthrough = true;
            };
          };
          # Also accept TLS passthrough on port 443 for the Matrix host so
          # other servers that contact :443/_matrix/*
          # will be forwarded to Conduit as well.
          # I guess I had to declare the rule explicitly here again?
          matrix-tls443 = {
            # match by SNI to avoid interfering with other HTTPS sites
            rule = "HostSNI(`lgbtq.prg-radio.org`)";
            service = "matrix-federation";
            entryPoints = ["websecure"];
            tls = {
              passthrough = true;
            };
          };

          # Mail TCP routers: SMTP (25), SMTPS (465 implicit TLS), IMAPS (993 implicit TLS)
          mail-smtp = {
            # catch-all TCP router for port 25
            rule = "HostSNI(`*`)";
            service = "mail-smtp";
            entryPoints = ["smtp"];
          };

          mail-smtps = {
            # SMTPS implicit TLS - passthrough to backend
            rule = "HostSNI(`*`)";
            service = "mail-smtps";
            entryPoints = ["smtps"];
            tls = {passthrough = true;};
          };

          mail-imaps = {
            # IMAPS implicit TLS - passthrough to backend
            rule = "HostSNI(`*`)";
            service = "mail-imaps";
            entryPoints = ["imaps"];
            tls = {passthrough = true;};
          };
        };

        tcp.services = {
          teamspeak-squery.loadBalancer = {
            servers = [
              {address = "10.1.1.248:10022";}
            ];
          };
          teamspeak-data.loadBalancer = {
            servers = [
              {address = "10.1.1.248:30033";}
            ];
          };
          matrix-federation.loadBalancer = {
            servers = [
              {address = "10.1.1.248:12244";}
            ];
          };

          # Mail TCP services
          mail-smtp.loadBalancer = {
            proxyProtocol = {version = 2;}; # Add this line
            servers = [{address = "10.1.1.15:25";}];
          };

          mail-smtps.loadBalancer = {
            proxyProtocol = {version = 2;}; # Add this line
            servers = [{address = "10.1.1.15:465";}];
          };

          mail-imaps.loadBalancer = {
            proxyProtocol = {version = 2;}; # Add this line
            servers = [{address = "10.1.1.15:993";}];
          };
        };

        # UDP routing for TeamSpeak voice
        udp.routers = {
          teamspeak-voice = {
            entryPoints = ["teamspeak-voice"];
            service = "teamspeak-voice";
          };
        };

        udp.services = {
          teamspeak-voice.loadBalancer = {
            servers = [
              {address = "10.1.1.248:9987";}
            ];
          };
        };
      };
    };

    security.acme = {
      acceptTerms = true;
      defaults.email = "dtu.prg@gmail.com";
      certs."prg-radio.org" = {
        domain = "*.prg-radio.org";
        group = "acme";
        dnsProvider = "cloudflare";
        environmentFile = "/home/traefikprg/cloudflare/cloudflare.env";
        reloadServices = ["traefik.service"];
      };
    };

    # PRG Cloudflare DDNS updater
    environment.etc."cloudflare-ddns/update.sh" = {
      text = let
        domain = "prg-radio.org";
        records = ["git" "grafana" "anubis" "wavelog" "partdb" "mail" "mailadmin" "@"];
        recordsStr = lib.concatStringsSep " " records;
        zoneId = "9fde8d0fa53502f2d1b7e0b1d3765d49";
        envFile = "/home/traefikprg/cloudflare/cloudflare.env";
      in ''
                #!/usr/bin/env bash
                set -euuo pipefail

                # Load environment variables (must contain CLOUDFLARE_API_TOKEN)
                if [ -f "${envFile}" ]; then
                  # shellcheck disable=SC1090
                  source "${envFile}"
                fi

                if [ -z "$CLOUDFLARE_API_TOKEN" ]; then
                  echo "CLOUDFLARE_API_TOKEN is not set in ${envFile}, aborting" >&2
                  exit 2
                fi

                API_BASE="https://api.cloudflare.com/client/v4"

                # get current public IPv4
                CURRENT_IP=$(curl -4 -sS https://ipv4.icanhazip.com | tr -d '\n') || true
                if [ -z "$CURRENT_IP" ]; then
                  echo "Failed to detect public IPv4 address" >&2
                  exit 3
                fi

                jq=${pkgs.jq}/bin/jq

                update_or_create() {
                  local fqdn="$1"
                  # Query existing A records for fqdn
                  res=$(curl -sS -X GET "$API_BASE/zones/${zoneId}/dns_records?type=A&name=$fqdn" \
                    -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \
                    -H "Content-Type: application/json")

                  ok=$(echo "$res" | $jq -r '.success')
                  if [ "$ok" != "true" ]; then
                    echo "Cloudflare API query failed for $fqdn: $res" >&2
                    return 1
                  fi

                  existing_ip=$(echo "$res" | $jq -r '.result[0].content // empty')
                  record_id=$(echo "$res" | $jq -r '.result[0].id // empty')

                  if [ -n "$existing_ip" ] && [ "$existing_ip" = "$CURRENT_IP" ]; then
                    echo "$fqdn is already set to $CURRENT_IP, skipping"
                    return 0
                  fi

                  payload=$(cat <<JSON
        {"type":"A","name":"$fqdn","content":"$CURRENT_IP","ttl":1,"proxied":false}
        JSON
        )

                  if [ -n "$record_id" ]; then
                    echo "Updating $fqdn ($record_id) -> $CURRENT_IP"
                    curl -sS -X PUT "$API_BASE/zones/${zoneId}/dns_records/$record_id" \
                      -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \
                      -H "Content-Type: application/json" \
                      --data "$payload" | $jq -r '.'
                  else
                    echo "Creating record $fqdn -> $CURRENT_IP"
                    curl -sS -X POST "$API_BASE/zones/${zoneId}/dns_records" \
                      -H "Authorization: Bearer $CLOUDFLARE_API_TOKEN" \
                      -H "Content-Type: application/json" \
                      --data "$payload" | $jq -r '.'
                  fi
                }

                echo "Starting PRG Cloudflare DDNS updater: setting IP=$CURRENT_IP"
                for r in ${recordsStr}; do
                  if [ "$r" = "@" ]; then
                    fqdn="${domain}"
                  else
                    fqdn="$r.${domain}"
                  fi
                  update_or_create "$fqdn" || true
                done
      '';
      mode = "0755";
      group = "root";
    };

    systemd.services."prg-cloudflare-ddns-updater" = {
      description = "PRG Cloudflare DDNS updater";
      wantedBy = ["multi-user.target"];
      serviceConfig = {
        Type = "oneshot";
        EnvironmentFile = "/home/traefikprg/cloudflare/cloudflare.env";
        ExecStart = "${pkgs.bash}/bin/bash /etc/cloudflare-ddns/update.sh";
      };
    };

    systemd.timers."prg-cloudflare-ddns-updater" = {
      description = "Run PRG Cloudflare DDNS updater periodically";
      wantedBy = ["timers.target"];
      timerConfig = {
        OnBootSec = "1m";
        OnUnitActiveSec = "10m";
        Persistent = true;
      };
    };

    # Enable Tailscale for remote access to Traefik dashboard and configuration
    services.tailscale.enable = true;

    systemd.services.traefik = {
      after = ["acme-finished-prg-radio.org.target"];
      wants = ["acme-finished-prg-radio.org.target"];
    };

    networking.firewall.allowedTCPPorts = [80 443 10022 30033 8448 9001 3001 993 465 25];
    networking.firewall.allowedUDPPorts = [80 443 9987 9001 3001 993 465 25];

    system.stateVersion = "25.11";
  };
}