the_prg_server_configuration_old/nix-system-configs/modules/system/gramethus.nix

{
  config,
  pkgs,
  lib,
  ...
}: let
  choose = paths: lib.findFirst builtins.pathExists null paths;

  # Dummy filesystem config for dry-run evaluation
  dummyFileSystems = {
    fileSystems."/" = {
      device = "/dev/disk/by-label/nixos";
      fsType = "ext4";
    };
  };
in {
  options.local = {
    hostname = lib.mkOption {
      type = lib.types.str;
      default = "nixos-default";
      description = "System hostname";
    };
    username = lib.mkOption {
      type = lib.types.str;
      default = "user";
      description = "Primary user username";
    };
    userDescription = lib.mkOption {
      type = lib.types.str;
      default = "NixOS User";
      description = "Primary user description";
    };
    address = lib.mkOption {
      type = lib.types.str;
      default = "10.1.1.100";
      description = "Static IP address";
    };
  };

  imports =
    lib.filter (x: x != null) [
      (choose [./modules/local/hostname_username.nix ../local/hostname_username.nix])
      (choose [./modules/local/networking_local.nix ../local/networking_local.nix])
      (choose [./modules/bootloader/seabios-assigned-iso-at-birth.nix ../bootloader/seabios-assigned-iso-at-birth.nix])
      (choose [./modules/lix-default.nix ../lix-default.nix])
      (choose [./modules/secrets-config/sops-the-blank-system.nix ../secrets-config/sops-the-blank-system.nix])
    ]
    # TODO: Make this more clean, migrate over to seabios-assigned-iso-at-birth.nix?
    #++ lib.optional (builtins.pathExists ./hardware-configuration.nix) ./hardware-configuration.nix
    #++ lib.optional (!builtins.pathExists ./hardware-configuration.nix) dummyFileSystems
    ;
  config = {
    local.hostname = "nixos-grametheus";
    local.username = "prglogs";
    local.userDescription = "NixOS PRG Grafana Prometheus Service";
    local.address = "10.1.1.10";

    system.stateVersion = "25.11";

    services.grafana = {
      enable = true;
      settings = {
        server = {
          http_addr = "100.71.227.33";
          http_port = 3000;
          #enforce_domain = true;
          enable_gzip = true;
          domain = "grafana.prg-radio.org";

          # Alternatively, if you want to serve Grafana from a subpath:
          # domain = "your.domain";
          # root_url = "https://your.domain/grafana/";
          # serve_from_sub_path = true;
        };

        # Prevents Grafana from phoning home
        #analytics.reporting_enabled = false;
      };
    };

    services.prometheus = {
      enable = true;
      port = 9001;

      exporters.node = {
        enabledCollectors = [
          "ethtool"
          "softirqs"
          "systemd"
          "tcpstat"
        ];
        enable = true;
        port = 9002;
      };
      globalConfig.scrape_interval = "10s"; # "1m"
      scrapeConfigs = [
        {
          job_name = "node";
          static_configs = [
            {
              targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
            }
          ];
        }
      ];
    };

    services.loki = {
      enable = true;
      configuration = {
        auth_enabled = false;

        server = {
          http_listen_port = 3100;
        };

        ingester = {
          lifecycler = {
            address = "0.0.0.0";
            ring = {
              kvstore = {
                store = "inmemory";
              };
              replication_factor = 1;
            };
            final_sleep = "0s";
          };
          chunk_idle_period = "1h";
          max_chunk_age = "1h";
          chunk_target_size = 1048576;
          chunk_retain_period = "30s";
        };

        schema_config = {
          configs = [
            {
              from = "2020-10-24";
              store = "boltdb-shipper";
              object_store = "filesystem";
              schema = "v11";
              index = {
                prefix = "index_";
                period = "24h";
              };
            }
          ];
        };

        storage_config = {
          boltdb_shipper = {
            active_index_directory = "/var/lib/loki/boltdb-shipper-active";
            cache_location = "/var/lib/loki/boltdb-shipper-cache";
            cache_ttl = "24h";
          };
          filesystem = {
            directory = "/var/lib/loki/chunks";
          };
        };

        limits_config = {
          reject_old_samples = true;
          reject_old_samples_max_age = "168h";
          # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used.
          allow_structured_metadata = false;
        };

        table_manager = {
          retention_deletes_enabled = false;
          retention_period = "0s";
        };
      };
    };

    services.alloy = {
      enable = true;
    };

    # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md
    environment.etc."alloy/config.alloy" = {
      text = ''
        discovery.relabel "journal" {
        	targets = []

        	rule {
        		source_labels = ["__journal__systemd_unit"]
        		target_label  = "unit"
        	}
        }

        loki.source.journal "journal" {
        	max_age       = "12h0m0s"
        	relabel_rules = discovery.relabel.journal.rules
        	forward_to    = [loki.write.default.receiver]
        	labels        = {
        		host = "chrysalis",
        		job  = "systemd-journal",
        	}
        }

        loki.write "default" {
        	endpoint {
        		url = "http://127.0.0.1:3100/loki/api/v1/push"
        	}
        	external_labels = {}
        }
      '';
      group = "root";
      mode = "0644";
    };

    # or alternatively
    #configFile = ./loki-config.yaml;
    # };

    # Enable Tailscale for remote access to Traefik dashboard and configuration
    services.tailscale.enable = true;

    networking.firewall.allowedTCPPorts = [3000];
    networking.firewall.allowedUDPPorts = [3000];
  };
}