the_prg_server_configuration_old/nix-system-configs/modules/toolsets/grafana_metric.nix

{
  config,
  pkgs,
  lib,
  ...
}: {
  services.prometheus = {
    enable = true;
    port = 9001;

    exporters.node = {
      enabledCollectors = [
        "ethtool"
        "softirqs"
        "systemd"
        "tcpstat"
      ];
      enable = true;
      port = 9002;
    };
    globalConfig.scrape_interval = "10s"; # "1m"
    scrapeConfigs = [
      {
        job_name = "node";
        static_configs = [
          {
            targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
          }
        ];
      }
    ];
  };

  services.loki = {
    enable = true;
    configuration = {
      auth_enabled = false;

      server = {
        http_listen_port = 3100;
      };

      ingester = {
        lifecycler = {
          address = "0.0.0.0";
          ring = {
            kvstore = {
              store = "inmemory";
            };
            replication_factor = 1;
          };
          final_sleep = "0s";
        };
        chunk_idle_period = "1h";
        max_chunk_age = "1h";
        chunk_target_size = 1048576;
        chunk_retain_period = "30s";
      };

      schema_config = {
        configs = [
          {
            from = "2020-10-24";
            store = "boltdb-shipper";
            object_store = "filesystem";
            schema = "v11";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }
        ];
      };

      storage_config = {
        boltdb_shipper = {
          active_index_directory = "/var/lib/loki/boltdb-shipper-active";
          cache_location = "/var/lib/loki/boltdb-shipper-cache";
          cache_ttl = "24h";
        };
        filesystem = {
          directory = "/var/lib/loki/chunks";
        };
      };

      limits_config = {
        reject_old_samples = true;
        reject_old_samples_max_age = "168h";
        # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used.
        allow_structured_metadata = false;
      };

      table_manager = {
        retention_deletes_enabled = false;
        retention_period = "0s";
      };
    };
  };

  services.alloy = {
    enable = true;
  };

  # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md
  environment.etc."alloy/config.alloy" = {
    text = ''
      discovery.relabel "journal" {
      	targets = []

      	rule {
      		source_labels = ["__journal__systemd_unit"]
      		target_label  = "unit"
      	}
      }

      loki.source.journal "journal" {
      	max_age       = "12h0m0s"
      	relabel_rules = discovery.relabel.journal.rules
      	forward_to    = [loki.write.default.receiver]
      	labels        = {
      		host = "chrysalis",
      		job  = "systemd-journal",
      	}
      }

      loki.write "default" {
      	endpoint {
      		url = "http://127.0.0.1:3100/loki/api/v1/push"
      	}
      	external_labels = {}
      }
    '';
    group = "root";
    mode = "0644";
  };

  networking.firewall.allowedTCPPorts = [3100 9001];
  networking.firewall.allowedUDPPorts = [3100 9001];

  systemd.services."loki-create-dirs" = {
    description = "Create Loki runtime/storage directories";
    after = ["network.target"];
    serviceConfig = {
      Type = "oneshot";
      ExecStart = ''        ${pkgs.bash}/bin/bash -c "set -eu; \
               mkdir -p /var/loki /var/loki/compactor
      '';
    };
    wantedBy = ["multi-user.target"];
  };
}