Added initial infrastructure for declarative prometheus configs and established alloy integration for Traefik

nix-system-configs/modules/system/gramethus.nix

@@ -217,5 +217,7 @@ in {
 
     networking.firewall.allowedTCPPorts = [3000];
     networking.firewall.allowedUDPPorts = [3000];
+
+    # ============== Declarative Config ================
   };
 }

nix-system-configs/modules/system/traefik.nix

@@ -93,10 +93,47 @@ in {
       };
     };
 
+    services.alloy = {
+      enable = true;
+    };
+
+    # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md
+    environment.etc."alloy/config.alloy" = {
+      text = ''
+        discovery.relabel "metrics_integrations_integrations_traefik" {
+        	targets = [{
+        		__address__ = "localhost:8080",
+        	}]
+
+        	rule {
+        		target_label = "instance"
+        		replacement  = constants.hostname
+        	}
+        }
+
+        prometheus.scrape "metrics_integrations_integrations_traefik" {
+        	targets    = discovery.relabel.metrics_integrations_integrations_traefik.output
+        	forward_to = [prometheus.remote_write.metrics_service.receiver]
+        	job_name   = "integrations/traefik"
+        }
+      '';
+      group = "root";
+      mode = "0644";
+    };
+
     services.traefik = {
       enable = true;
       group = "acme";
       staticConfigOptions = {
+        metrics = {
+          prometheus = {
+            addEntryPointsLabels = true;
+            addRoutersLabels = true;
+            addServicesLabels = true;
+            entryPoint = "metrics";
+          };
+        };
+
         entryPoints = {
           web = {
             # Bind on IPv6 as well as IPv4 by using [::]:80

nix-system-configs/modules/toolsets/grafana_metric.nix

@@ -0,0 +1,139 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: {
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+
+    exporters.node = {
+      enabledCollectors = [
+        "ethtool"
+        "softirqs"
+        "systemd"
+        "tcpstat"
+      ];
+      enable = true;
+      port = 9002;
+    };
+    globalConfig.scrape_interval = "10s"; # "1m"
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [
+          {
+            targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
+          }
+        ];
+      }
+    ];
+  };
+
+  services.loki = {
+    enable = true;
+    configuration = {
+      auth_enabled = false;
+
+      server = {
+        http_listen_port = 3100;
+      };
+
+      ingester = {
+        lifecycler = {
+          address = "0.0.0.0";
+          ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+            replication_factor = 1;
+          };
+          final_sleep = "0s";
+        };
+        chunk_idle_period = "1h";
+        max_chunk_age = "1h";
+        chunk_target_size = 1048576;
+        chunk_retain_period = "30s";
+      };
+
+      schema_config = {
+        configs = [
+          {
+            from = "2020-10-24";
+            store = "boltdb-shipper";
+            object_store = "filesystem";
+            schema = "v11";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
+          cache_location = "/var/lib/loki/boltdb-shipper-cache";
+          cache_ttl = "24h";
+        };
+        filesystem = {
+          directory = "/var/lib/loki/chunks";
+        };
+      };
+
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+        # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used.
+        allow_structured_metadata = false;
+      };
+
+      table_manager = {
+        retention_deletes_enabled = false;
+        retention_period = "0s";
+      };
+    };
+  };
+
+  services.alloy = {
+    enable = true;
+  };
+
+  # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md
+  environment.etc."alloy/config.alloy" = {
+    text = ''
+      discovery.relabel "journal" {
+      	targets = []
+
+      	rule {
+      		source_labels = ["__journal__systemd_unit"]
+      		target_label  = "unit"
+      	}
+      }
+
+      loki.source.journal "journal" {
+      	max_age       = "12h0m0s"
+      	relabel_rules = discovery.relabel.journal.rules
+      	forward_to    = [loki.write.default.receiver]
+      	labels        = {
+      		host = "chrysalis",
+      		job  = "systemd-journal",
+      	}
+      }
+
+      loki.write "default" {
+      	endpoint {
+      		url = "http://127.0.0.1:3100/loki/api/v1/push"
+      	}
+      	external_labels = {}
+      }
+    '';
+    group = "root";
+    mode = "0644";
+  };
+
+  networking.firewall.allowedTCPPorts = [3100 9001];
+  networking.firewall.allowedUDPPorts = [3100 9001];
+}