From e6b7a95b57fc34c35acd7a5219829daf1febef4f Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 14:34:24 +0100
Subject: [PATCH 01/27] Add nixos-logs system with Grafana, Prometheus, and
 Loki; fix Traefik config formatting

---
 dry_run.zsh                                   |   1 +
 flake.nix                                     |   5 +
 .../modules/system/gramethus.nix              | 110 ++++++++++++++++++
 nix-system-configs/modules/system/traefik.nix |  21 ++--
 4 files changed, 126 insertions(+), 11 deletions(-)
 create mode 100644 nix-system-configs/modules/system/gramethus.nix

diff --git a/dry_run.zsh b/dry_run.zsh
index db5f6c5..9920b80 100755
--- a/dry_run.zsh
+++ b/dry_run.zsh
@@ -15,6 +15,7 @@ configs=(
   "nixos-teamspeak"
   "nixos-traefik"
   "nixos-build-machine"
+  "nixos-logs"
 )
 
 if [[ -n "$1" ]]; then
diff --git a/flake.nix b/flake.nix
index ad1ebd7..c43d220 100644
--- a/flake.nix
+++ b/flake.nix
@@ -52,6 +52,11 @@
         inherit system;
         modules = [./nix-system-configs/modules/system/build_machine.nix];
       };
+
+      "nixos-logs" = nixpkgs.lib.nixosSystem {
+        inherit system;
+        modules = [./nix-system-configs/modules/system/gramethus.nix];
+      };
     };
   };
 }
diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
new file mode 100644
index 0000000..8ccce3a
--- /dev/null
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -0,0 +1,110 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  choose = paths: lib.findFirst builtins.pathExists null paths;
+
+  # Dummy filesystem config for dry-run evaluation
+  dummyFileSystems = {
+    fileSystems."/" = {
+      device = "/dev/disk/by-label/nixos";
+      fsType = "ext4";
+    };
+  };
+in {
+  options.local = {
+    hostname = lib.mkOption {
+      type = lib.types.str;
+      default = "nixos-default";
+      description = "System hostname";
+    };
+    username = lib.mkOption {
+      type = lib.types.str;
+      default = "user";
+      description = "Primary user username";
+    };
+    userDescription = lib.mkOption {
+      type = lib.types.str;
+      default = "NixOS User";
+      description = "Primary user description";
+    };
+    address = lib.mkOption {
+      type = lib.types.str;
+      default = "10.1.1.100";
+      description = "Static IP address";
+    };
+  };
+
+  imports =
+    lib.filter (x: x != null) [
+      (choose [./modules/local/hostname_username.nix ../local/hostname_username.nix])
+      (choose [./modules/local/networking_local.nix ../local/networking_local.nix])
+      (choose [./modules/bootloader/seabios-assigned-iso-at-birth.nix ../bootloader/seabios-assigned-iso-at-birth.nix])
+      (choose [./modules/lix-default.nix ../lix-default.nix])
+      (choose [./modules/secrets-config/sops-the-blank-system.nix ../secrets-config/sops-the-blank-system.nix])
+    ]
+    # TODO: Make this more clean, migrate over to seabios-assigned-iso-at-birth.nix?
+    #++ lib.optional (builtins.pathExists ./hardware-configuration.nix) ./hardware-configuration.nix
+    #++ lib.optional (!builtins.pathExists ./hardware-configuration.nix) dummyFileSystems
+    ;
+  config = {
+    local.hostname = "nixos-grametheus";
+    local.username = "prglogs";
+    local.userDescription = "NixOS PRG Grafana Prometheus Service";
+    local.address = "10.1.1.10";
+
+    system.stateVersion = "25.11";
+
+    services.grafana = {
+      enable = true;
+      settings = {
+        server = {
+          http_addr = "127.0.0.1";
+          http_port = 3005;
+          enforce_domain = true;
+          enable_gzip = true;
+          domain = "grafana.prg-radio.org";
+
+          # Alternatively, if you want to serve Grafana from a subpath:
+          # domain = "your.domain";
+          # root_url = "https://your.domain/grafana/";
+          # serve_from_sub_path = true;
+        };
+
+        # Prevents Grafana from phoning home
+        #analytics.reporting_enabled = false;
+      };
+    };
+
+    services.prometheus = {
+      enable = true;
+      port = 9001;
+
+      exporters.node.enabledCollectors = [
+        "ethtool"
+        "softirqs"
+        "systemd"
+        "tcpstat"
+      ];
+    };
+
+    services.loki = {
+      enable = true;
+
+      configuration = {
+        auth_enabled = false;
+        server.http_listen_port = 3100;
+
+        # etc.
+      };
+
+      # or alternatively
+      #configFile = ./loki-config.yaml;
+    };
+
+    # Enable Tailscale for remote access to Traefik dashboard and configuration
+    services.tailscale.enable = true;
+  };
+}
diff --git a/nix-system-configs/modules/system/traefik.nix b/nix-system-configs/modules/system/traefik.nix
index 0d7d0d0..f87af3d 100644
--- a/nix-system-configs/modules/system/traefik.nix
+++ b/nix-system-configs/modules/system/traefik.nix
@@ -78,7 +78,7 @@ in {
 
             # Prevents infinite loop of doom:
             # See: https://github.com/TecharoHQ/anubis/issues/970
-            JWT_RESTRICTION_HEADER="CF-Connecting-IP";
+            JWT_RESTRICTION_HEADER = "CF-Connecting-IP";
 
             # Metrics on separate port
             METRICS_BIND_NETWORK = "tcp";
@@ -90,7 +90,6 @@ in {
             COOKIE_SAME_SITE = "None"; # Only if Secure=true and you need cross-site
           };
         };
-
       };
     };
 
@@ -174,9 +173,9 @@ in {
           anubis = {
             rule = "Host(`anubis.prg-radio.org`)";
             service = "anubis";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {
-                certresolver = "acme";
+              certresolver = "acme";
             };
           };
 
@@ -184,16 +183,16 @@ in {
           forgejo = {
             rule = "Host(`git.prg-radio.org`)";
             service = "forgejo";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {};
-            middlewares = [ "anubisForwardAuth" ];
+            middlewares = ["anubisForwardAuth"];
           };
 
           # Matrix HTTP router for client requests (Element etc.)
           matrix = {
             rule = "Host(`lgbtq.prg-radio.org`)";
             service = "matrix";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {};
           };
 
@@ -201,7 +200,7 @@ in {
           wavelog = {
             rule = "Host(`wavelog.prg-radio.org`)";
             service = "wavelog";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {};
           };
 
@@ -209,9 +208,9 @@ in {
           partdb = {
             rule = "Host(`partdb.prg-radio.org`)";
             service = "partdb";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {};
-            middlewares = [ "anubisForwardAuth" ];
+            middlewares = ["anubisForwardAuth"];
           };
         };
 
@@ -344,4 +343,4 @@ in {
 
     system.stateVersion = "25.11";
   };
-}
\ No newline at end of file
+}

From 35e5b04ad8a9fa8c472abd28f186c03b47151e7c Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 14:40:10 +0100
Subject: [PATCH 02/27] Temporarily disable Loki

---
 .../modules/system/gramethus.nix              | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index 8ccce3a..5c3baa2 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -90,19 +90,19 @@ in {
       ];
     };
 
-    services.loki = {
-      enable = true;
+    #services.loki = {
+    #   enable = true;
+    #
+    #  configuration = {
+    #    auth_enabled = false;
+    #     server.http_listen_port = 3100;
+    #
+    #     # etc.
+    #   };
 
-      configuration = {
-        auth_enabled = false;
-        server.http_listen_port = 3100;
-
-        # etc.
-      };
-
-      # or alternatively
-      #configFile = ./loki-config.yaml;
-    };
+    # or alternatively
+    #configFile = ./loki-config.yaml;
+    # };
 
     # Enable Tailscale for remote access to Traefik dashboard and configuration
     services.tailscale.enable = true;

From 058fc0b6672b3b4536ee7c25167025171ac968a7 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 14:50:17 +0100
Subject: [PATCH 03/27] Expose Grafana on port 3005 and add Traefik route with
 Anubis auth

---
 nix-system-configs/modules/system/gramethus.nix |  4 +++-
 nix-system-configs/modules/system/traefik.nix   | 15 +++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index 5c3baa2..f672023 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -61,7 +61,6 @@ in {
       enable = true;
       settings = {
         server = {
-          http_addr = "127.0.0.1";
           http_port = 3005;
           enforce_domain = true;
           enable_gzip = true;
@@ -106,5 +105,8 @@ in {
 
     # Enable Tailscale for remote access to Traefik dashboard and configuration
     services.tailscale.enable = true;
+
+    networking.firewall.allowedTCPPorts = [3005];
+    networking.firewall.allowedUDPPorts = [3005];
   };
 }
diff --git a/nix-system-configs/modules/system/traefik.nix b/nix-system-configs/modules/system/traefik.nix
index f87af3d..84005a5 100644
--- a/nix-system-configs/modules/system/traefik.nix
+++ b/nix-system-configs/modules/system/traefik.nix
@@ -212,6 +212,15 @@ in {
             tls = {};
             middlewares = ["anubisForwardAuth"];
           };
+
+          # Protected service: Grafana
+          grafana = {
+            rule = "Host(`grafana.prg-radio.org`)";
+            service = "grafana";
+            entryPoints = ["websecure"];
+            tls = {};
+            middlewares = ["anubisForwardAuth"];
+          };
         };
 
         http.services = {
@@ -243,6 +252,12 @@ in {
               {url = "http://10.1.1.249:8087";}
             ];
           };
+
+          grafana.loadBalancer = {
+            servers = [
+              {url = "http://10.1.1.10:3005";}
+            ];
+          };
         };
 
         # TCP routing for TeamSpeak

From ff8ad949137c6f57d7c9b298dc96e1db9cee1076 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 15:25:25 +0100
Subject: [PATCH 04/27] Configure Promethus Scrape Intefval

---
 nix-system-configs/modules/system/gramethus.nix | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index f672023..70c809c 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -61,8 +61,9 @@ in {
       enable = true;
       settings = {
         server = {
-          http_port = 3005;
-          enforce_domain = true;
+          http_addr = "100.71.227.33";
+          http_port = 3000;
+          #enforce_domain = true;
           enable_gzip = true;
           domain = "grafana.prg-radio.org";
 
@@ -87,6 +88,18 @@ in {
         "systemd"
         "tcpstat"
       ];
+
+      globalConfig.scrape_interval = "10s"; # "1m"
+      scrapeConfigs = [
+        {
+          job_name = "node";
+          static_configs = [
+            {
+              targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
+            }
+          ];
+        }
+      ];
     };
 
     #services.loki = {

From 0e902d7156020ed98cd6eca1fd791f7c7e440d87 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 15:36:14 +0100
Subject: [PATCH 05/27] Play around with config options.

---
 .../modules/system/gramethus.nix              | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index 70c809c..d44c3cb 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -79,16 +79,16 @@ in {
     };
 
     services.prometheus = {
-      enable = true;
-      port = 9001;
-
-      exporters.node.enabledCollectors = [
-        "ethtool"
-        "softirqs"
-        "systemd"
-        "tcpstat"
-      ];
-
+      exporters.node = {
+        enabledCollectors = [
+          "ethtool"
+          "softirqs"
+          "systemd"
+          "tcpstat"
+        ];
+        enable = true;
+        port = 9001;
+      };
       globalConfig.scrape_interval = "10s"; # "1m"
       scrapeConfigs = [
         {

From f771863d97d39b3b8d0d0f49e5c0f159284cdcdd Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 15:48:11 +0100
Subject: [PATCH 06/27] Oops, now I know more how Prometheus works.

---
 nix-system-configs/modules/system/gramethus.nix | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index d44c3cb..ed96a8c 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -79,6 +79,9 @@ in {
     };
 
     services.prometheus = {
+      enable = true;
+      port = 9001;
+
       exporters.node = {
         enabledCollectors = [
           "ethtool"
@@ -87,7 +90,7 @@ in {
           "tcpstat"
         ];
         enable = true;
-        port = 9001;
+        port = 9002;
       };
       globalConfig.scrape_interval = "10s"; # "1m"
       scrapeConfigs = [

From c65057ded3440cccbe9f2ff83927efd6ea406bf4 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 16:25:46 +0100
Subject: [PATCH 07/27] Grafana Alloy and Loki Additions.

---
 .../modules/system/gramethus.nix              | 81 ++++++++++++++++---
 1 file changed, 70 insertions(+), 11 deletions(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index ed96a8c..046186e 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -13,6 +13,38 @@
       fsType = "ext4";
     };
   };
+
+  environment.etc."alloy/config.alloy" = {
+    text = ''
+      server:
+        http_listen_port: 28183
+        grpc_listen_port: 0
+
+      logs:
+        # keep positions persistent on disk (don't use /tmp in production)
+        positions:
+          filename: /var/lib/grafana-agent/positions.yaml
+
+        configs:
+          - name: journal
+            journal:
+              # keep similar rotation / lookback to your promtail config
+              max_age: 12h
+              labels:
+                job: systemd-journal
+                host: chrysalis
+
+            clients:
+              - url: http://127.0.0.1:3100/loki/api/v1/push
+
+            relabel_configs:
+              - source_labels: ["__journal__systemd_unit"]
+                target_label: "unit"
+    '';
+    owner = "root";
+    group = "root";
+    mode = "0644";
+  };
 in {
   options.local = {
     hostname = lib.mkOption {
@@ -105,15 +137,42 @@ in {
       ];
     };
 
-    #services.loki = {
-    #   enable = true;
-    #
-    #  configuration = {
-    #    auth_enabled = false;
-    #     server.http_listen_port = 3100;
-    #
-    #     # etc.
-    #   };
+    services.loki = {
+      enable = true;
+
+      configuration = {
+        auth_enabled = false;
+        server.http_listen_port = 3100;
+
+        # etc.
+      };
+    };
+
+    services.alloy = {
+      enable = true;
+    };
+
+    # Add a simple systemd service to run the Grafana Agent (Alloy)
+    systemd.services.grafana-agent = {
+      description = "Grafana Alloy  - journal -> loki";
+      wantedBy = ["multi-user.target"];
+
+      serviceConfig = {
+        # Ensure persistent dir for positions exists
+        ExecStartPre = ''
+          mkdir -p /var/lib/grafana-agent
+          chown root:root /var/lib/grafana-agent
+        '';
+
+        # The grafana-agent binary provided by nixpkgs
+        ExecStart = ''
+          ${pkgs.grafana-alloy}/bin/grafana-alloy --config.file /etc/alloy/config.alloy
+        '';
+
+        Restart = "on-failure";
+        RestartSec = "5s";
+      };
+    };
 
     # or alternatively
     #configFile = ./loki-config.yaml;
@@ -122,7 +181,7 @@ in {
     # Enable Tailscale for remote access to Traefik dashboard and configuration
     services.tailscale.enable = true;
 
-    networking.firewall.allowedTCPPorts = [3005];
-    networking.firewall.allowedUDPPorts = [3005];
+    networking.firewall.allowedTCPPorts = [3000];
+    networking.firewall.allowedUDPPorts = [3000];
   };
 }

From 85029b850703800841a7afee1099e17b6d7f7025 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 16:32:30 +0100
Subject: [PATCH 08/27] Fix the Loki config

---
 .../modules/system/gramethus.nix              | 65 ++++++++++++++++++-
 1 file changed, 62 insertions(+), 3 deletions(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index 046186e..b13ce2f 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -139,12 +139,71 @@ in {
 
     services.loki = {
       enable = true;
-
       configuration = {
         auth_enabled = false;
-        server.http_listen_port = 3100;
 
-        # etc.
+        server = {
+          http_listen_port = 3100;
+        };
+
+        ingester = {
+          lifecycler = {
+            address = "0.0.0.0";
+            ring = {
+              kvstore = {
+                store = "inmemory";
+              };
+              replication_factor = 1;
+            };
+            final_sleep = "0s";
+          };
+          chunk_idle_period = "1h";
+          max_chunk_age = "1h";
+          chunk_target_size = 1048576;
+          chunk_retain_period = "30s";
+          max_transfer_retries = 0;
+        };
+
+        schema_config = {
+          configs = [
+            {
+              from = "2020-10-24";
+              store = "boltdb-shipper";
+              object_store = "filesystem";
+              schema = "v11";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+        };
+
+        storage_config = {
+          boltdb_shipper = {
+            active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+            cache_location = "/var/lib/loki/boltdb-shipper-cache";
+            cache_ttl = "24h";
+            shared_store = "filesystem";
+          };
+          filesystem = {
+            directory = "/var/lib/loki/chunks";
+          };
+        };
+
+        limits_config = {
+          reject_old_samples = true;
+          reject_old_samples_max_age = "168h";
+        };
+
+        chunk_store_config = {
+          max_look_back_period = "0s";
+        };
+
+        table_manager = {
+          retention_deletes_enabled = false;
+          retention_period = "0s";
+        };
       };
     };
 

From 9bf4acd3bdcab58e0d8fbf7492a4e11825818a13 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 16:34:58 +0100
Subject: [PATCH 09/27] Remove unused and redundant Loki configuration options,
 why dry run does not find these.

---
 nix-system-configs/modules/system/gramethus.nix | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index b13ce2f..bd87eaa 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -161,7 +161,6 @@ in {
           max_chunk_age = "1h";
           chunk_target_size = 1048576;
           chunk_retain_period = "30s";
-          max_transfer_retries = 0;
         };
 
         schema_config = {
@@ -184,7 +183,6 @@ in {
             active_index_directory = "/var/lib/loki/boltdb-shipper-active";
             cache_location = "/var/lib/loki/boltdb-shipper-cache";
             cache_ttl = "24h";
-            shared_store = "filesystem";
           };
           filesystem = {
             directory = "/var/lib/loki/chunks";
@@ -196,9 +194,6 @@ in {
           reject_old_samples_max_age = "168h";
         };
 
-        chunk_store_config = {
-          max_look_back_period = "0s";
-        };
 
         table_manager = {
           retention_deletes_enabled = false;

From 9363dc3b364c6663b925246407518494dc147c39 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 16:36:56 +0100
Subject: [PATCH 10/27] Odd issues... odd issues every where.

---
 nix-system-configs/modules/system/gramethus.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index bd87eaa..05cbead 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -192,6 +192,8 @@ in {
         limits_config = {
           reject_old_samples = true;
           reject_old_samples_max_age = "168h";
+          # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used.
+          allow_structured_metadata = false;
         };
 
 

From 21bf4848bb3dd028dc2f11d0bc1efdf5d0e0766b Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 16:48:24 +0100
Subject: [PATCH 11/27] Fix the Grafana Alloy

---
 nix-system-configs/modules/system/gramethus.nix | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index 05cbead..d282ded 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -210,19 +210,13 @@ in {
 
     # Add a simple systemd service to run the Grafana Agent (Alloy)
     systemd.services.grafana-agent = {
-      description = "Grafana Alloy  - journal -> loki";
+      description = "Grafana Alloy";
       wantedBy = ["multi-user.target"];
 
       serviceConfig = {
-        # Ensure persistent dir for positions exists
-        ExecStartPre = ''
-          mkdir -p /var/lib/grafana-agent
-          chown root:root /var/lib/grafana-agent
-        '';
-
         # The grafana-agent binary provided by nixpkgs
         ExecStart = ''
-          ${pkgs.grafana-alloy}/bin/grafana-alloy --config.file /etc/alloy/config.alloy
+          ${pkgs.grafana-alloy}/bin/alloy --config.file /etc/alloy/config.alloy
         '';
 
         Restart = "on-failure";

From 6bd1d4a231e7ed056b99168ab717ce8e44ef273f Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 16:52:24 +0100
Subject: [PATCH 12/27] Fix the broken environment call

---
 nix-system-configs/modules/system/gramethus.nix | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index d282ded..1b73109 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -14,7 +14,7 @@
     };
   };
 
-  environment.etc."alloy/config.alloy" = {
+  environment.etc.alloy.config.alloy = {
     text = ''
       server:
         http_listen_port: 28183
@@ -216,7 +216,7 @@ in {
       serviceConfig = {
         # The grafana-agent binary provided by nixpkgs
         ExecStart = ''
-          ${pkgs.grafana-alloy}/bin/alloy --config.file /etc/alloy/config.alloy
+          ${pkgs.grafana-alloy}/bin/alloy
         '';
 
         Restart = "on-failure";

From 458e08596463c9b1fc6d9d191e852d4afc3af0f4 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 16:54:45 +0100
Subject: [PATCH 13/27] Derp by me - put the allow config call elsehwere it
 should be.

---
 .../modules/system/gramethus.nix              | 67 +++++++++----------
 1 file changed, 33 insertions(+), 34 deletions(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index 1b73109..32ea262 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -13,38 +13,6 @@
       fsType = "ext4";
     };
   };
-
-  environment.etc.alloy.config.alloy = {
-    text = ''
-      server:
-        http_listen_port: 28183
-        grpc_listen_port: 0
-
-      logs:
-        # keep positions persistent on disk (don't use /tmp in production)
-        positions:
-          filename: /var/lib/grafana-agent/positions.yaml
-
-        configs:
-          - name: journal
-            journal:
-              # keep similar rotation / lookback to your promtail config
-              max_age: 12h
-              labels:
-                job: systemd-journal
-                host: chrysalis
-
-            clients:
-              - url: http://127.0.0.1:3100/loki/api/v1/push
-
-            relabel_configs:
-              - source_labels: ["__journal__systemd_unit"]
-                target_label: "unit"
-    '';
-    owner = "root";
-    group = "root";
-    mode = "0644";
-  };
 in {
   options.local = {
     hostname = lib.mkOption {
@@ -196,7 +164,6 @@ in {
           allow_structured_metadata = false;
         };
 
-
         table_manager = {
           retention_deletes_enabled = false;
           retention_period = "0s";
@@ -208,6 +175,38 @@ in {
       enable = true;
     };
 
+    environment.etc.alloy.config.alloy = {
+      text = ''
+        server:
+          http_listen_port: 28183
+          grpc_listen_port: 0
+
+        logs:
+          # keep positions persistent on disk (don't use /tmp in production)
+          positions:
+            filename: /var/lib/grafana-agent/positions.yaml
+
+          configs:
+            - name: journal
+              journal:
+                # keep similar rotation / lookback to your promtail config
+                max_age: 12h
+                labels:
+                  job: systemd-journal
+                  host: chrysalis
+
+              clients:
+                - url: http://127.0.0.1:3100/loki/api/v1/push
+
+              relabel_configs:
+                - source_labels: ["__journal__systemd_unit"]
+                  target_label: "unit"
+      '';
+      owner = "root";
+      group = "root";
+      mode = "0644";
+    };
+
     # Add a simple systemd service to run the Grafana Agent (Alloy)
     systemd.services.grafana-agent = {
       description = "Grafana Alloy";
@@ -216,7 +215,7 @@ in {
       serviceConfig = {
         # The grafana-agent binary provided by nixpkgs
         ExecStart = ''
-          ${pkgs.grafana-alloy}/bin/alloy
+          ${pkgs.grafana-alloy}/bin/alloy run
         '';
 
         Restart = "on-failure";

From 75bf74d31986b269c8908d54c224fd4010b4e203 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 16:57:24 +0100
Subject: [PATCH 14/27] insane suffixes, fixed the broken suffix

---
 nix-system-configs/modules/system/gramethus.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index 32ea262..bcfa73a 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -175,7 +175,7 @@ in {
       enable = true;
     };
 
-    environment.etc.alloy.config.alloy = {
+    environment.etc."alloy/config.alloy" = {
       text = ''
         server:
           http_listen_port: 28183

From a519b8ffec24ce3a64375871eccdd3bab080700a Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 16:58:54 +0100
Subject: [PATCH 15/27] Dangling wrong config.

---
 nix-system-configs/modules/system/gramethus.nix | 1 -
 1 file changed, 1 deletion(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index bcfa73a..6e4ddbf 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -202,7 +202,6 @@ in {
                 - source_labels: ["__journal__systemd_unit"]
                   target_label: "unit"
       '';
-      owner = "root";
       group = "root";
       mode = "0644";
     };

From 99c43c145e0e142997dec88ec803fb827b72eaa1 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 17:09:50 +0100
Subject: [PATCH 16/27] Wrong format..

---
 .../modules/system/gramethus.nix              | 63 ++++++++-----------
 1 file changed, 27 insertions(+), 36 deletions(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index 6e4ddbf..105fcf7 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -177,51 +177,42 @@ in {
 
     environment.etc."alloy/config.alloy" = {
       text = ''
-        server:
-          http_listen_port: 28183
-          grpc_listen_port: 0
+        loki.relabel "journal" {
+          rule {
+            source_labels = ["__journal__systemd_unit"]
+            target_label  = "unit"
+          }
+        }
 
-        logs:
-          # keep positions persistent on disk (don't use /tmp in production)
-          positions:
-            filename: /var/lib/grafana-agent/positions.yaml
+        loki.write "local" {
+          endpoint {
+            url = "http://127.0.0.1:3100/loki/api/v1/push"
+          }
+          external_labels = {}
+        }
 
-          configs:
-            - name: journal
-              journal:
-                # keep similar rotation / lookback to your promtail config
-                max_age: 12h
-                labels:
-                  job: systemd-journal
-                  host: chrysalis
+        loki.source.journal "journal" {
+          max_age = "12h0m0s"
 
-              clients:
-                - url: http://127.0.0.1:3100/loki/api/v1/push
+          forward_to = [loki.write.local.receiver]
 
-              relabel_configs:
-                - source_labels: ["__journal__systemd_unit"]
-                  target_label: "unit"
+          labels = {
+            job  = "systemd-journal"
+            host = "chrysalis"
+          }
+
+          relabel_rules = loki.relabel.journal.rules
+
+          legacy_position {
+            file = "/var/lib/grafana-agent/positions.yaml"
+            name = "journal"
+          }
+        }
       '';
       group = "root";
       mode = "0644";
     };
 
-    # Add a simple systemd service to run the Grafana Agent (Alloy)
-    systemd.services.grafana-agent = {
-      description = "Grafana Alloy";
-      wantedBy = ["multi-user.target"];
-
-      serviceConfig = {
-        # The grafana-agent binary provided by nixpkgs
-        ExecStart = ''
-          ${pkgs.grafana-alloy}/bin/alloy run
-        '';
-
-        Restart = "on-failure";
-        RestartSec = "5s";
-      };
-    };
-
     # or alternatively
     #configFile = ./loki-config.yaml;
     # };

From c11896c50c04a993349d53811ed98c9eca43fa99 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 17:19:21 +0100
Subject: [PATCH 17/27] Properly convert document as per
 https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md

---
 .../modules/system/gramethus.nix              | 45 +++++++++----------
 1 file changed, 20 insertions(+), 25 deletions(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index 105fcf7..a6924fd 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -175,38 +175,33 @@ in {
       enable = true;
     };
 
+    # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md
     environment.etc."alloy/config.alloy" = {
       text = ''
-        loki.relabel "journal" {
-          rule {
-            source_labels = ["__journal__systemd_unit"]
-            target_label  = "unit"
-          }
-        }
+        discovery.relabel "journal" {
+        	targets = []
 
-        loki.write "local" {
-          endpoint {
-            url = "http://127.0.0.1:3100/loki/api/v1/push"
-          }
-          external_labels = {}
+        	rule {
+        		source_labels = ["__journal__systemd_unit"]
+        		target_label  = "unit"
+        	}
         }
 
         loki.source.journal "journal" {
-          max_age = "12h0m0s"
+        	max_age       = "12h0m0s"
+        	relabel_rules = discovery.relabel.journal.rules
+        	forward_to    = [loki.write.default.receiver]
+        	labels        = {
+        		host = "chrysalis",
+        		job  = "systemd-journal",
+        	}
+        }
 
-          forward_to = [loki.write.local.receiver]
-
-          labels = {
-            job  = "systemd-journal"
-            host = "chrysalis"
-          }
-
-          relabel_rules = loki.relabel.journal.rules
-
-          legacy_position {
-            file = "/var/lib/grafana-agent/positions.yaml"
-            name = "journal"
-          }
+        loki.write "default" {
+        	endpoint {
+        		url = "http://127.0.0.1:3100/loki/api/v1/push"
+        	}
+        	external_labels = {}
         }
       '';
       group = "root";

From 6704bdcdfd336615c092b0500fa6a7ba8830aff7 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 17:27:41 +0100
Subject: [PATCH 18/27] Update addresses.

---
 nix-system-configs/modules/system/gramethus.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index a6924fd..3dd63e3 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -61,7 +61,7 @@ in {
       enable = true;
       settings = {
         server = {
-          http_addr = "100.71.227.33";
+          http_addr = "10.1.1.10";
           http_port = 3000;
           #enforce_domain = true;
           enable_gzip = true;

From 19c7f994ede08be1b57b6353bcd6fad296f12346 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 17:28:41 +0100
Subject: [PATCH 19/27] Update port of Grafana.

---
 nix-system-configs/modules/system/traefik.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-system-configs/modules/system/traefik.nix b/nix-system-configs/modules/system/traefik.nix
index 84005a5..908d32e 100644
--- a/nix-system-configs/modules/system/traefik.nix
+++ b/nix-system-configs/modules/system/traefik.nix
@@ -255,7 +255,7 @@ in {
 
           grafana.loadBalancer = {
             servers = [
-              {url = "http://10.1.1.10:3005";}
+              {url = "http://10.1.1.10:3000";}
             ];
           };
         };

From d5920fb8fa1474a6345ada5b4fe52d7f87c23d02 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 17:33:22 +0100
Subject: [PATCH 20/27] Averted the risk of lack of redirects.

---
 nix-system-configs/modules/system/traefik.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-system-configs/modules/system/traefik.nix b/nix-system-configs/modules/system/traefik.nix
index 908d32e..cd4f717 100644
--- a/nix-system-configs/modules/system/traefik.nix
+++ b/nix-system-configs/modules/system/traefik.nix
@@ -58,7 +58,7 @@ in {
             TARGET = " ";
 
             # Configure redirect domains - ALL domains that should be able to redirect back after challenge
-            REDIRECT_DOMAINS = "prg-radio.org,anubis.prg-radio.org,git.prg-radio.org,lgbtq.prg-radio.org,wavelog.prg-radio.org,partdb.prg-radio.org";
+            REDIRECT_DOMAINS = "prg-radio.org,anubis.prg-radio.org,git.prg-radio.org,lgbtq.prg-radio.org,wavelog.prg-radio.org,partdb.prg-radio.org,grafana.prg-radio.org";
 
             # Public URL for Anubis
             PUBLIC_URL = "https://anubis.prg-radio.org";

From 2665a2c0e542f56beeda9f2ab0cf268336f102ab Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 18:49:10 +0100
Subject: [PATCH 21/27] Added initial infrastructure for declarative prometheus
 configs and established alloy integration for Traefik

---
 .../modules/system/gramethus.nix              |   2 +
 nix-system-configs/modules/system/traefik.nix |  37 +++++
 .../modules/toolsets/grafana_metric.nix       | 139 ++++++++++++++++++
 3 files changed, 178 insertions(+)
 create mode 100644 nix-system-configs/modules/toolsets/grafana_metric.nix

diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix
index 3dd63e3..2a08dae 100644
--- a/nix-system-configs/modules/system/gramethus.nix
+++ b/nix-system-configs/modules/system/gramethus.nix
@@ -217,5 +217,7 @@ in {
 
     networking.firewall.allowedTCPPorts = [3000];
     networking.firewall.allowedUDPPorts = [3000];
+
+    # ============== Declarative Config ================
   };
 }
diff --git a/nix-system-configs/modules/system/traefik.nix b/nix-system-configs/modules/system/traefik.nix
index cd4f717..4dc5240 100644
--- a/nix-system-configs/modules/system/traefik.nix
+++ b/nix-system-configs/modules/system/traefik.nix
@@ -93,10 +93,47 @@ in {
       };
     };
 
+    services.alloy = {
+      enable = true;
+    };
+
+    # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md
+    environment.etc."alloy/config.alloy" = {
+      text = ''
+        discovery.relabel "metrics_integrations_integrations_traefik" {
+        	targets = [{
+        		__address__ = "localhost:8080",
+        	}]
+
+        	rule {
+        		target_label = "instance"
+        		replacement  = constants.hostname
+        	}
+        }
+
+        prometheus.scrape "metrics_integrations_integrations_traefik" {
+        	targets    = discovery.relabel.metrics_integrations_integrations_traefik.output
+        	forward_to = [prometheus.remote_write.metrics_service.receiver]
+        	job_name   = "integrations/traefik"
+        }
+      '';
+      group = "root";
+      mode = "0644";
+    };
+
     services.traefik = {
       enable = true;
       group = "acme";
       staticConfigOptions = {
+        metrics = {
+          prometheus = {
+            addEntryPointsLabels = true;
+            addRoutersLabels = true;
+            addServicesLabels = true;
+            entryPoint = "metrics";
+          };
+        };
+
         entryPoints = {
           web = {
             # Bind on IPv6 as well as IPv4 by using [::]:80
diff --git a/nix-system-configs/modules/toolsets/grafana_metric.nix b/nix-system-configs/modules/toolsets/grafana_metric.nix
new file mode 100644
index 0000000..5cc95f9
--- /dev/null
+++ b/nix-system-configs/modules/toolsets/grafana_metric.nix
@@ -0,0 +1,139 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: {
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+
+    exporters.node = {
+      enabledCollectors = [
+        "ethtool"
+        "softirqs"
+        "systemd"
+        "tcpstat"
+      ];
+      enable = true;
+      port = 9002;
+    };
+    globalConfig.scrape_interval = "10s"; # "1m"
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [
+          {
+            targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
+          }
+        ];
+      }
+    ];
+  };
+
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+
+      server = {
+        http_listen_port = 3100;
+      };
+
+      ingester = {
+        lifecycler = {
+          address = "0.0.0.0";
+          ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+            replication_factor = 1;
+          };
+          final_sleep = "0s";
+        };
+        chunk_idle_period = "1h";
+        max_chunk_age = "1h";
+        chunk_target_size = 1048576;
+        chunk_retain_period = "30s";
+      };
+
+      schema_config = {
+        configs = [
+          {
+            from = "2020-10-24";
+            store = "boltdb-shipper";
+            object_store = "filesystem";
+            schema = "v11";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+          cache_ttl = "24h";
+        };
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+        # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used.
+        allow_structured_metadata = false;
+      };
+
+      table_manager = {
+        retention_deletes_enabled = false;
+        retention_period = "0s";
+      };
+    };
+  };
+
+  services.alloy = {
+    enable = true;
+  };
+
+  # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md
+  environment.etc."alloy/config.alloy" = {
+    text = ''
+      discovery.relabel "journal" {
+      	targets = []
+
+      	rule {
+      		source_labels = ["__journal__systemd_unit"]
+      		target_label  = "unit"
+      	}
+      }
+
+      loki.source.journal "journal" {
+      	max_age       = "12h0m0s"
+      	relabel_rules = discovery.relabel.journal.rules
+      	forward_to    = [loki.write.default.receiver]
+      	labels        = {
+      		host = "chrysalis",
+      		job  = "systemd-journal",
+      	}
+      }
+
+      loki.write "default" {
+      	endpoint {
+      		url = "http://127.0.0.1:3100/loki/api/v1/push"
+      	}
+      	external_labels = {}
+      }
+    '';
+    group = "root";
+    mode = "0644";
+  };
+
+  networking.firewall.allowedTCPPorts = [3100 9001];
+  networking.firewall.allowedUDPPorts = [3100 9001];
+}

From f7f0039f9cad9fd9fd9c1213a9db8364f2363ed3 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 19:00:12 +0100
Subject: [PATCH 22/27] Fixed some prometheus settings.

---
 nix-system-configs/modules/system/traefik.nix | 27 +++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/nix-system-configs/modules/system/traefik.nix b/nix-system-configs/modules/system/traefik.nix
index 4dc5240..bed085e 100644
--- a/nix-system-configs/modules/system/traefik.nix
+++ b/nix-system-configs/modules/system/traefik.nix
@@ -97,6 +97,33 @@ in {
       enable = true;
     };
 
+    services.prometheus = {
+      enable = true;
+      port = 9001;
+
+      exporters.node = {
+        enabledCollectors = [
+          "ethtool"
+          "softirqs"
+          "systemd"
+          "tcpstat"
+        ];
+        enable = true;
+        port = 9002;
+      };
+      globalConfig.scrape_interval = "5s"; # "1m"
+      scrapeConfigs = [
+        {
+          job_name = "traefik";
+          static_configs = [
+            {
+              targets = ["localhost:8080"];
+            }
+          ];
+        }
+      ];
+    };
+
     # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md
     environment.etc."alloy/config.alloy" = {
       text = ''

From ba2cf5c149083e1e2e5bd1ff51ac88fea1d75589 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 19:02:24 +0100
Subject: [PATCH 23/27] Fixed some prometheus settings... again.

---
 nix-system-configs/modules/system/traefik.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/nix-system-configs/modules/system/traefik.nix b/nix-system-configs/modules/system/traefik.nix
index bed085e..81b887c 100644
--- a/nix-system-configs/modules/system/traefik.nix
+++ b/nix-system-configs/modules/system/traefik.nix
@@ -140,7 +140,7 @@ in {
 
         prometheus.scrape "metrics_integrations_integrations_traefik" {
         	targets    = discovery.relabel.metrics_integrations_integrations_traefik.output
-        	forward_to = [prometheus.remote_write.metrics_service.receiver]
+        	# forward_to = [prometheus.remote_write.metrics_service.receiver]
         	job_name   = "integrations/traefik"
         }
       '';
@@ -417,8 +417,8 @@ in {
       wants = ["acme-finished-prg-radio.org.target"];
     };
 
-    networking.firewall.allowedTCPPorts = [80 443 10022 30033 8448];
-    networking.firewall.allowedUDPPorts = [80 443 9987];
+    networking.firewall.allowedTCPPorts = [80 443 10022 30033 8448 9001];
+    networking.firewall.allowedUDPPorts = [80 443 9987 9001];
 
     system.stateVersion = "25.11";
   };

From bcca1141b6dac399e8e6c8233cc749293ebb6a35 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 19:03:33 +0100
Subject: [PATCH 24/27] Another fix...

---
 nix-system-configs/modules/system/traefik.nix | 1 -
 1 file changed, 1 deletion(-)

diff --git a/nix-system-configs/modules/system/traefik.nix b/nix-system-configs/modules/system/traefik.nix
index 81b887c..b14e28f 100644
--- a/nix-system-configs/modules/system/traefik.nix
+++ b/nix-system-configs/modules/system/traefik.nix
@@ -140,7 +140,6 @@ in {
 
         prometheus.scrape "metrics_integrations_integrations_traefik" {
         	targets    = discovery.relabel.metrics_integrations_integrations_traefik.output
-        	# forward_to = [prometheus.remote_write.metrics_service.receiver]
         	job_name   = "integrations/traefik"
         }
       '';

From b4abbfaa27246a2dd448c2a35ce5ce3c29b0bb6b Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 19:12:47 +0100
Subject: [PATCH 25/27] Playing around settings..

---
 nix-system-configs/modules/system/traefik.nix | 71 ++++++++++++++++++-
 1 file changed, 69 insertions(+), 2 deletions(-)

diff --git a/nix-system-configs/modules/system/traefik.nix b/nix-system-configs/modules/system/traefik.nix
index b14e28f..dfdae36 100644
--- a/nix-system-configs/modules/system/traefik.nix
+++ b/nix-system-configs/modules/system/traefik.nix
@@ -93,6 +93,72 @@ in {
       };
     };
 
+    services.loki = {
+      enable = true;
+      configuration = {
+        auth_enabled = false;
+
+        server = {
+          http_listen_port = 3100;
+        };
+
+        ingester = {
+          lifecycler = {
+            address = "0.0.0.0";
+            ring = {
+              kvstore = {
+                store = "inmemory";
+              };
+              replication_factor = 1;
+            };
+            final_sleep = "0s";
+          };
+          chunk_idle_period = "1h";
+          max_chunk_age = "1h";
+          chunk_target_size = 1048576;
+          chunk_retain_period = "30s";
+        };
+
+        schema_config = {
+          configs = [
+            {
+              from = "2020-10-24";
+              store = "boltdb-shipper";
+              object_store = "filesystem";
+              schema = "v11";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+        };
+
+        storage_config = {
+          boltdb_shipper = {
+            active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+            cache_location = "/var/lib/loki/boltdb-shipper-cache";
+            cache_ttl = "24h";
+          };
+          filesystem = {
+            directory = "/var/lib/loki/chunks";
+          };
+        };
+
+        limits_config = {
+          reject_old_samples = true;
+          reject_old_samples_max_age = "168h";
+          # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used.
+          allow_structured_metadata = false;
+        };
+
+        table_manager = {
+          retention_deletes_enabled = false;
+          retention_period = "0s";
+        };
+      };
+    };
+
     services.alloy = {
       enable = true;
     };
@@ -140,6 +206,7 @@ in {
 
         prometheus.scrape "metrics_integrations_integrations_traefik" {
         	targets    = discovery.relabel.metrics_integrations_integrations_traefik.output
+            forward_to = [ ]
         	job_name   = "integrations/traefik"
         }
       '';
@@ -416,8 +483,8 @@ in {
       wants = ["acme-finished-prg-radio.org.target"];
     };
 
-    networking.firewall.allowedTCPPorts = [80 443 10022 30033 8448 9001];
-    networking.firewall.allowedUDPPorts = [80 443 9987 9001];
+    networking.firewall.allowedTCPPorts = [80 443 10022 30033 8448 9001 3001];
+    networking.firewall.allowedUDPPorts = [80 443 9987 9001 3001];
 
     system.stateVersion = "25.11";
   };

From f84861132a7fef05cca4ecd64b354054468d092a Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 19:32:48 +0100
Subject: [PATCH 26/27] Re-enable the fowarding.

---
 nix-system-configs/modules/system/traefik.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix-system-configs/modules/system/traefik.nix b/nix-system-configs/modules/system/traefik.nix
index dfdae36..cc3389d 100644
--- a/nix-system-configs/modules/system/traefik.nix
+++ b/nix-system-configs/modules/system/traefik.nix
@@ -206,7 +206,7 @@ in {
 
         prometheus.scrape "metrics_integrations_integrations_traefik" {
         	targets    = discovery.relabel.metrics_integrations_integrations_traefik.output
-            forward_to = [ ]
+            forward_to = [ prometheus.remote_write.metrics_service.receiver ]
         	job_name   = "integrations/traefik"
         }
       '';

From 7bbacc8512b8aec4b7d505145fa5da50cc5706a6 Mon Sep 17 00:00:00 2001
From: Christine Elisabeth Koppel <kuutruu@posteo.net>
Date: Sat, 14 Feb 2026 20:10:54 +0100
Subject: [PATCH 27/27] Try to diagnose Traefik metrics issue.

---
 nix-system-configs/modules/system/traefik.nix | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/nix-system-configs/modules/system/traefik.nix b/nix-system-configs/modules/system/traefik.nix
index cc3389d..a458f3a 100644
--- a/nix-system-configs/modules/system/traefik.nix
+++ b/nix-system-configs/modules/system/traefik.nix
@@ -206,7 +206,6 @@ in {
 
         prometheus.scrape "metrics_integrations_integrations_traefik" {
         	targets    = discovery.relabel.metrics_integrations_integrations_traefik.output
-            forward_to = [ prometheus.remote_write.metrics_service.receiver ]
         	job_name   = "integrations/traefik"
         }
       '';
@@ -223,7 +222,7 @@ in {
             addEntryPointsLabels = true;
             addRoutersLabels = true;
             addServicesLabels = true;
-            entryPoint = "metrics";
+            buckets = [0.1 0.3 1.2 5.0];
           };
         };