Add nixos-logs system with Grafana, Prometheus, and Loki; fix Traefik config formatting

dry_run.zsh

@@ -15,6 +15,7 @@ configs=(
   "nixos-teamspeak"
   "nixos-traefik"
   "nixos-build-machine"
+  "nixos-logs"
 )
 
 if [[ -n "$1" ]]; then

flake.nix

@@ -52,6 +52,11 @@
         inherit system;
         modules = [./nix-system-configs/modules/system/build_machine.nix];
       };
+
+      "nixos-logs" = nixpkgs.lib.nixosSystem {
+        inherit system;
+        modules = [./nix-system-configs/modules/system/gramethus.nix];
+      };
     };
   };
 }

nix-system-configs/modules/system/gramethus.nix

@@ -0,0 +1,110 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  choose = paths: lib.findFirst builtins.pathExists null paths;
+
+  # Dummy filesystem config for dry-run evaluation
+  dummyFileSystems = {
+    fileSystems."/" = {
+      device = "/dev/disk/by-label/nixos";
+      fsType = "ext4";
+    };
+  };
+in {
+  options.local = {
+    hostname = lib.mkOption {
+      type = lib.types.str;
+      default = "nixos-default";
+      description = "System hostname";
+    };
+    username = lib.mkOption {
+      type = lib.types.str;
+      default = "user";
+      description = "Primary user username";
+    };
+    userDescription = lib.mkOption {
+      type = lib.types.str;
+      default = "NixOS User";
+      description = "Primary user description";
+    };
+    address = lib.mkOption {
+      type = lib.types.str;
+      default = "10.1.1.100";
+      description = "Static IP address";
+    };
+  };
+
+  imports =
+    lib.filter (x: x != null) [
+      (choose [./modules/local/hostname_username.nix ../local/hostname_username.nix])
+      (choose [./modules/local/networking_local.nix ../local/networking_local.nix])
+      (choose [./modules/bootloader/seabios-assigned-iso-at-birth.nix ../bootloader/seabios-assigned-iso-at-birth.nix])
+      (choose [./modules/lix-default.nix ../lix-default.nix])
+      (choose [./modules/secrets-config/sops-the-blank-system.nix ../secrets-config/sops-the-blank-system.nix])
+    ]
+    # TODO: Make this more clean, migrate over to seabios-assigned-iso-at-birth.nix?
+    #++ lib.optional (builtins.pathExists ./hardware-configuration.nix) ./hardware-configuration.nix
+    #++ lib.optional (!builtins.pathExists ./hardware-configuration.nix) dummyFileSystems
+    ;
+  config = {
+    local.hostname = "nixos-grametheus";
+    local.username = "prglogs";
+    local.userDescription = "NixOS PRG Grafana Prometheus Service";
+    local.address = "10.1.1.10";
+
+    system.stateVersion = "25.11";
+
+    services.grafana = {
+      enable = true;
+      settings = {
+        server = {
+          http_addr = "127.0.0.1";
+          http_port = 3005;
+          enforce_domain = true;
+          enable_gzip = true;
+          domain = "grafana.prg-radio.org";
+
+          # Alternatively, if you want to serve Grafana from a subpath:
+          # domain = "your.domain";
+          # root_url = "https://your.domain/grafana/";
+          # serve_from_sub_path = true;
+        };
+
+        # Prevents Grafana from phoning home
+        #analytics.reporting_enabled = false;
+      };
+    };
+
+    services.prometheus = {
+      enable = true;
+      port = 9001;
+
+      exporters.node.enabledCollectors = [
+        "ethtool"
+        "softirqs"
+        "systemd"
+        "tcpstat"
+      ];
+    };
+
+    services.loki = {
+      enable = true;
+
+      configuration = {
+        auth_enabled = false;
+        server.http_listen_port = 3100;
+
+        # etc.
+      };
+
+      # or alternatively
+      #configFile = ./loki-config.yaml;
+    };
+
+    # Enable Tailscale for remote access to Traefik dashboard and configuration
+    services.tailscale.enable = true;
+  };
+}

nix-system-configs/modules/system/traefik.nix

@@ -78,7 +78,7 @@ in {
 
             # Prevents infinite loop of doom:
             # See: https://github.com/TecharoHQ/anubis/issues/970
-            JWT_RESTRICTION_HEADER="CF-Connecting-IP";
+            JWT_RESTRICTION_HEADER = "CF-Connecting-IP";
 
             # Metrics on separate port
             METRICS_BIND_NETWORK = "tcp";
@@ -90,7 +90,6 @@ in {
             COOKIE_SAME_SITE = "None"; # Only if Secure=true and you need cross-site
           };
         };
-
       };
     };
 
@@ -174,9 +173,9 @@ in {
           anubis = {
             rule = "Host(`anubis.prg-radio.org`)";
             service = "anubis";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {
-                certresolver = "acme";
+              certresolver = "acme";
             };
           };
 
@@ -184,16 +183,16 @@ in {
           forgejo = {
             rule = "Host(`git.prg-radio.org`)";
             service = "forgejo";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {};
-            middlewares = [ "anubisForwardAuth" ];
+            middlewares = ["anubisForwardAuth"];
           };
 
           # Matrix HTTP router for client requests (Element etc.)
           matrix = {
             rule = "Host(`lgbtq.prg-radio.org`)";
             service = "matrix";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {};
           };
 
@@ -201,7 +200,7 @@ in {
           wavelog = {
             rule = "Host(`wavelog.prg-radio.org`)";
             service = "wavelog";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {};
           };
 
@@ -209,9 +208,9 @@ in {
           partdb = {
             rule = "Host(`partdb.prg-radio.org`)";
             service = "partdb";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {};
-            middlewares = [ "anubisForwardAuth" ];
+            middlewares = ["anubisForwardAuth"];
           };
         };