Try to diagnose Traefik metrics issue.

dry_run.zsh

@@ -15,6 +15,7 @@ configs=(
   "nixos-teamspeak"
   "nixos-traefik"
   "nixos-build-machine"
+  "nixos-logs"
 )
 
 if [[ -n "$1" ]]; then

flake.nix

@@ -52,6 +52,11 @@
         inherit system;
         modules = [./nix-system-configs/modules/system/build_machine.nix];
       };
+
+      "nixos-logs" = nixpkgs.lib.nixosSystem {
+        inherit system;
+        modules = [./nix-system-configs/modules/system/gramethus.nix];
+      };
     };
   };
 }

nix-system-configs/modules/system/gramethus.nix

@@ -0,0 +1,223 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  choose = paths: lib.findFirst builtins.pathExists null paths;
+
+  # Dummy filesystem config for dry-run evaluation
+  dummyFileSystems = {
+    fileSystems."/" = {
+      device = "/dev/disk/by-label/nixos";
+      fsType = "ext4";
+    };
+  };
+in {
+  options.local = {
+    hostname = lib.mkOption {
+      type = lib.types.str;
+      default = "nixos-default";
+      description = "System hostname";
+    };
+    username = lib.mkOption {
+      type = lib.types.str;
+      default = "user";
+      description = "Primary user username";
+    };
+    userDescription = lib.mkOption {
+      type = lib.types.str;
+      default = "NixOS User";
+      description = "Primary user description";
+    };
+    address = lib.mkOption {
+      type = lib.types.str;
+      default = "10.1.1.100";
+      description = "Static IP address";
+    };
+  };
+
+  imports =
+    lib.filter (x: x != null) [
+      (choose [./modules/local/hostname_username.nix ../local/hostname_username.nix])
+      (choose [./modules/local/networking_local.nix ../local/networking_local.nix])
+      (choose [./modules/bootloader/seabios-assigned-iso-at-birth.nix ../bootloader/seabios-assigned-iso-at-birth.nix])
+      (choose [./modules/lix-default.nix ../lix-default.nix])
+      (choose [./modules/secrets-config/sops-the-blank-system.nix ../secrets-config/sops-the-blank-system.nix])
+    ]
+    # TODO: Make this more clean, migrate over to seabios-assigned-iso-at-birth.nix?
+    #++ lib.optional (builtins.pathExists ./hardware-configuration.nix) ./hardware-configuration.nix
+    #++ lib.optional (!builtins.pathExists ./hardware-configuration.nix) dummyFileSystems
+    ;
+  config = {
+    local.hostname = "nixos-grametheus";
+    local.username = "prglogs";
+    local.userDescription = "NixOS PRG Grafana Prometheus Service";
+    local.address = "10.1.1.10";
+
+    system.stateVersion = "25.11";
+
+    services.grafana = {
+      enable = true;
+      settings = {
+        server = {
+          http_addr = "10.1.1.10";
+          http_port = 3000;
+          #enforce_domain = true;
+          enable_gzip = true;
+          domain = "grafana.prg-radio.org";
+
+          # Alternatively, if you want to serve Grafana from a subpath:
+          # domain = "your.domain";
+          # root_url = "https://your.domain/grafana/";
+          # serve_from_sub_path = true;
+        };
+
+        # Prevents Grafana from phoning home
+        #analytics.reporting_enabled = false;
+      };
+    };
+
+    services.prometheus = {
+      enable = true;
+      port = 9001;
+
+      exporters.node = {
+        enabledCollectors = [
+          "ethtool"
+          "softirqs"
+          "systemd"
+          "tcpstat"
+        ];
+        enable = true;
+        port = 9002;
+      };
+      globalConfig.scrape_interval = "10s"; # "1m"
+      scrapeConfigs = [
+        {
+          job_name = "node";
+          static_configs = [
+            {
+              targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
+            }
+          ];
+        }
+      ];
+    };
+
+    services.loki = {
+      enable = true;
+      configuration = {
+        auth_enabled = false;
+
+        server = {
+          http_listen_port = 3100;
+        };
+
+        ingester = {
+          lifecycler = {
+            address = "0.0.0.0";
+            ring = {
+              kvstore = {
+                store = "inmemory";
+              };
+              replication_factor = 1;
+            };
+            final_sleep = "0s";
+          };
+          chunk_idle_period = "1h";
+          max_chunk_age = "1h";
+          chunk_target_size = 1048576;
+          chunk_retain_period = "30s";
+        };
+
+        schema_config = {
+          configs = [
+            {
+              from = "2020-10-24";
+              store = "boltdb-shipper";
+              object_store = "filesystem";
+              schema = "v11";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+        };
+
+        storage_config = {
+          boltdb_shipper = {
+            active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+            cache_location = "/var/lib/loki/boltdb-shipper-cache";
+            cache_ttl = "24h";
+          };
+          filesystem = {
+            directory = "/var/lib/loki/chunks";
+          };
+        };
+
+        limits_config = {
+          reject_old_samples = true;
+          reject_old_samples_max_age = "168h";
+          # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used.
+          allow_structured_metadata = false;
+        };
+
+        table_manager = {
+          retention_deletes_enabled = false;
+          retention_period = "0s";
+        };
+      };
+    };
+
+    services.alloy = {
+      enable = true;
+    };
+
+    # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md
+    environment.etc."alloy/config.alloy" = {
+      text = ''
+        discovery.relabel "journal" {
+        	targets = []
+
+        	rule {
+        		source_labels = ["__journal__systemd_unit"]
+        		target_label  = "unit"
+        	}
+        }
+
+        loki.source.journal "journal" {
+        	max_age       = "12h0m0s"
+        	relabel_rules = discovery.relabel.journal.rules
+        	forward_to    = [loki.write.default.receiver]
+        	labels        = {
+        		host = "chrysalis",
+        		job  = "systemd-journal",
+        	}
+        }
+
+        loki.write "default" {
+        	endpoint {
+        		url = "http://127.0.0.1:3100/loki/api/v1/push"
+        	}
+        	external_labels = {}
+        }
+      '';
+      group = "root";
+      mode = "0644";
+    };
+
+    # or alternatively
+    #configFile = ./loki-config.yaml;
+    # };
+
+    # Enable Tailscale for remote access to Traefik dashboard and configuration
+    services.tailscale.enable = true;
+
+    networking.firewall.allowedTCPPorts = [3000];
+    networking.firewall.allowedUDPPorts = [3000];
+
+    # ============== Declarative Config ================
+  };
+}

nix-system-configs/modules/system/traefik.nix

@@ -58,7 +58,7 @@ in {
             TARGET = " ";
 
             # Configure redirect domains - ALL domains that should be able to redirect back after challenge
-            REDIRECT_DOMAINS = "prg-radio.org,anubis.prg-radio.org,git.prg-radio.org,lgbtq.prg-radio.org,wavelog.prg-radio.org,partdb.prg-radio.org";
+            REDIRECT_DOMAINS = "prg-radio.org,anubis.prg-radio.org,git.prg-radio.org,lgbtq.prg-radio.org,wavelog.prg-radio.org,partdb.prg-radio.org,grafana.prg-radio.org";
 
             # Public URL for Anubis
             PUBLIC_URL = "https://anubis.prg-radio.org";
@@ -78,7 +78,7 @@ in {
 
             # Prevents infinite loop of doom:
             # See: https://github.com/TecharoHQ/anubis/issues/970
-            JWT_RESTRICTION_HEADER="CF-Connecting-IP";
+            JWT_RESTRICTION_HEADER = "CF-Connecting-IP";
 
             # Metrics on separate port
             METRICS_BIND_NETWORK = "tcp";
@@ -90,14 +90,142 @@ in {
             COOKIE_SAME_SITE = "None"; # Only if Secure=true and you need cross-site
           };
         };
-
       };
     };
 
+    services.loki = {
+      enable = true;
+      configuration = {
+        auth_enabled = false;
+
+        server = {
+          http_listen_port = 3100;
+        };
+
+        ingester = {
+          lifecycler = {
+            address = "0.0.0.0";
+            ring = {
+              kvstore = {
+                store = "inmemory";
+              };
+              replication_factor = 1;
+            };
+            final_sleep = "0s";
+          };
+          chunk_idle_period = "1h";
+          max_chunk_age = "1h";
+          chunk_target_size = 1048576;
+          chunk_retain_period = "30s";
+        };
+
+        schema_config = {
+          configs = [
+            {
+              from = "2020-10-24";
+              store = "boltdb-shipper";
+              object_store = "filesystem";
+              schema = "v11";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+        };
+
+        storage_config = {
+          boltdb_shipper = {
+            active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+            cache_location = "/var/lib/loki/boltdb-shipper-cache";
+            cache_ttl = "24h";
+          };
+          filesystem = {
+            directory = "/var/lib/loki/chunks";
+          };
+        };
+
+        limits_config = {
+          reject_old_samples = true;
+          reject_old_samples_max_age = "168h";
+          # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used.
+          allow_structured_metadata = false;
+        };
+
+        table_manager = {
+          retention_deletes_enabled = false;
+          retention_period = "0s";
+        };
+      };
+    };
+
+    services.alloy = {
+      enable = true;
+    };
+
+    services.prometheus = {
+      enable = true;
+      port = 9001;
+
+      exporters.node = {
+        enabledCollectors = [
+          "ethtool"
+          "softirqs"
+          "systemd"
+          "tcpstat"
+        ];
+        enable = true;
+        port = 9002;
+      };
+      globalConfig.scrape_interval = "5s"; # "1m"
+      scrapeConfigs = [
+        {
+          job_name = "traefik";
+          static_configs = [
+            {
+              targets = ["localhost:8080"];
+            }
+          ];
+        }
+      ];
+    };
+
+    # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md
+    environment.etc."alloy/config.alloy" = {
+      text = ''
+        discovery.relabel "metrics_integrations_integrations_traefik" {
+        	targets = [{
+        		__address__ = "localhost:8080",
+        	}]
+
+        	rule {
+        		target_label = "instance"
+        		replacement  = constants.hostname
+        	}
+        }
+
+        prometheus.scrape "metrics_integrations_integrations_traefik" {
+        	targets    = discovery.relabel.metrics_integrations_integrations_traefik.output
+        	job_name   = "integrations/traefik"
+        }
+      '';
+      group = "root";
+      mode = "0644";
+    };
+
     services.traefik = {
       enable = true;
       group = "acme";
       staticConfigOptions = {
+        metrics = {
+          prometheus = {
+            addEntryPointsLabels = true;
+            addRoutersLabels = true;
+            addServicesLabels = true;
+            buckets = [0.1 0.3 1.2 5.0];
+          };
+        };
+
         entryPoints = {
           web = {
             # Bind on IPv6 as well as IPv4 by using [::]:80
@@ -174,9 +302,9 @@ in {
           anubis = {
             rule = "Host(`anubis.prg-radio.org`)";
             service = "anubis";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {
-                certresolver = "acme";
+              certresolver = "acme";
             };
           };
 
@@ -184,16 +312,16 @@ in {
           forgejo = {
             rule = "Host(`git.prg-radio.org`)";
             service = "forgejo";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {};
-            middlewares = [ "anubisForwardAuth" ];
+            middlewares = ["anubisForwardAuth"];
           };
 
           # Matrix HTTP router for client requests (Element etc.)
           matrix = {
             rule = "Host(`lgbtq.prg-radio.org`)";
             service = "matrix";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {};
           };
 
@@ -201,7 +329,7 @@ in {
           wavelog = {
             rule = "Host(`wavelog.prg-radio.org`)";
             service = "wavelog";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {};
           };
 
@@ -209,9 +337,18 @@ in {
           partdb = {
             rule = "Host(`partdb.prg-radio.org`)";
             service = "partdb";
-            entryPoints = [ "websecure" ];
+            entryPoints = ["websecure"];
             tls = {};
-            middlewares = [ "anubisForwardAuth" ];
+            middlewares = ["anubisForwardAuth"];
+          };
+
+          # Protected service: Grafana
+          grafana = {
+            rule = "Host(`grafana.prg-radio.org`)";
+            service = "grafana";
+            entryPoints = ["websecure"];
+            tls = {};
+            middlewares = ["anubisForwardAuth"];
           };
         };
 
@@ -244,6 +381,12 @@ in {
               {url = "http://10.1.1.249:8087";}
             ];
           };
+
+          grafana.loadBalancer = {
+            servers = [
+              {url = "http://10.1.1.10:3000";}
+            ];
+          };
         };
 
         # TCP routing for TeamSpeak
@@ -339,9 +482,9 @@ in {
       wants = ["acme-finished-prg-radio.org.target"];
     };
 
-    networking.firewall.allowedTCPPorts = [80 443 10022 30033 8448];
-    networking.firewall.allowedUDPPorts = [80 443 9987];
+    networking.firewall.allowedTCPPorts = [80 443 10022 30033 8448 9001 3001];
+    networking.firewall.allowedUDPPorts = [80 443 9987 9001 3001];
 
     system.stateVersion = "25.11";
   };
-}
+}

nix-system-configs/modules/toolsets/grafana_metric.nix

@@ -0,0 +1,139 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: {
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+
+    exporters.node = {
+      enabledCollectors = [
+        "ethtool"
+        "softirqs"
+        "systemd"
+        "tcpstat"
+      ];
+      enable = true;
+      port = 9002;
+    };
+    globalConfig.scrape_interval = "10s"; # "1m"
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [
+          {
+            targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
+          }
+        ];
+      }
+    ];
+  };
+
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+
+      server = {
+        http_listen_port = 3100;
+      };
+
+      ingester = {
+        lifecycler = {
+          address = "0.0.0.0";
+          ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+            replication_factor = 1;
+          };
+          final_sleep = "0s";
+        };
+        chunk_idle_period = "1h";
+        max_chunk_age = "1h";
+        chunk_target_size = 1048576;
+        chunk_retain_period = "30s";
+      };
+
+      schema_config = {
+        configs = [
+          {
+            from = "2020-10-24";
+            store = "boltdb-shipper";
+            object_store = "filesystem";
+            schema = "v11";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+          cache_ttl = "24h";
+        };
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+        # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used.
+        allow_structured_metadata = false;
+      };
+
+      table_manager = {
+        retention_deletes_enabled = false;
+        retention_period = "0s";
+      };
+    };
+  };
+
+  services.alloy = {
+    enable = true;
+  };
+
+  # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md
+  environment.etc."alloy/config.alloy" = {
+    text = ''
+      discovery.relabel "journal" {
+      	targets = []
+
+      	rule {
+      		source_labels = ["__journal__systemd_unit"]
+      		target_label  = "unit"
+      	}
+      }
+
+      loki.source.journal "journal" {
+      	max_age       = "12h0m0s"
+      	relabel_rules = discovery.relabel.journal.rules
+      	forward_to    = [loki.write.default.receiver]
+      	labels        = {
+      		host = "chrysalis",
+      		job  = "systemd-journal",
+      	}
+      }
+
+      loki.write "default" {
+      	endpoint {
+      		url = "http://127.0.0.1:3100/loki/api/v1/push"
+      	}
+      	external_labels = {}
+      }
+    '';
+    group = "root";
+    mode = "0644";
+  };
+
+  networking.firewall.allowedTCPPorts = [3100 9001];
+  networking.firewall.allowedUDPPorts = [3100 9001];
+}