the_prg_server_configuration_old/nix-system-configs/modules/system/gramethus.nix

{
  config,
  pkgs,
  lib,
  ...
}: let
  choose = paths: lib.findFirst builtins.pathExists null paths;

  # Dummy filesystem config for dry-run evaluation
  dummyFileSystems = {
    fileSystems."/" = {
      device = "/dev/disk/by-label/nixos";
      fsType = "ext4";
    };
  };

  environment.etc."alloy/config.alloy" = {
    text = ''
      server:
        http_listen_port: 28183
        grpc_listen_port: 0

      logs:
        # keep positions persistent on disk (don't use /tmp in production)
        positions:
          filename: /var/lib/grafana-agent/positions.yaml

        configs:
          - name: journal
            journal:
              # keep similar rotation / lookback to your promtail config
              max_age: 12h
              labels:
                job: systemd-journal
                host: chrysalis

            clients:
              - url: http://127.0.0.1:3100/loki/api/v1/push

            relabel_configs:
              - source_labels: ["__journal__systemd_unit"]
                target_label: "unit"
    '';
    owner = "root";
    group = "root";
    mode = "0644";
  };
in {
  options.local = {
    hostname = lib.mkOption {
      type = lib.types.str;
      default = "nixos-default";
      description = "System hostname";
    };
    username = lib.mkOption {
      type = lib.types.str;
      default = "user";
      description = "Primary user username";
    };
    userDescription = lib.mkOption {
      type = lib.types.str;
      default = "NixOS User";
      description = "Primary user description";
    };
    address = lib.mkOption {
      type = lib.types.str;
      default = "10.1.1.100";
      description = "Static IP address";
    };
  };

  imports =
    lib.filter (x: x != null) [
      (choose [./modules/local/hostname_username.nix ../local/hostname_username.nix])
      (choose [./modules/local/networking_local.nix ../local/networking_local.nix])
      (choose [./modules/bootloader/seabios-assigned-iso-at-birth.nix ../bootloader/seabios-assigned-iso-at-birth.nix])
      (choose [./modules/lix-default.nix ../lix-default.nix])
      (choose [./modules/secrets-config/sops-the-blank-system.nix ../secrets-config/sops-the-blank-system.nix])
    ]
    # TODO: Make this more clean, migrate over to seabios-assigned-iso-at-birth.nix?
    #++ lib.optional (builtins.pathExists ./hardware-configuration.nix) ./hardware-configuration.nix
    #++ lib.optional (!builtins.pathExists ./hardware-configuration.nix) dummyFileSystems
    ;
  config = {
    local.hostname = "nixos-grametheus";
    local.username = "prglogs";
    local.userDescription = "NixOS PRG Grafana Prometheus Service";
    local.address = "10.1.1.10";

    system.stateVersion = "25.11";

    services.grafana = {
      enable = true;
      settings = {
        server = {
          http_addr = "100.71.227.33";
          http_port = 3000;
          #enforce_domain = true;
          enable_gzip = true;
          domain = "grafana.prg-radio.org";

          # Alternatively, if you want to serve Grafana from a subpath:
          # domain = "your.domain";
          # root_url = "https://your.domain/grafana/";
          # serve_from_sub_path = true;
        };

        # Prevents Grafana from phoning home
        #analytics.reporting_enabled = false;
      };
    };

    services.prometheus = {
      enable = true;
      port = 9001;

      exporters.node = {
        enabledCollectors = [
          "ethtool"
          "softirqs"
          "systemd"
          "tcpstat"
        ];
        enable = true;
        port = 9002;
      };
      globalConfig.scrape_interval = "10s"; # "1m"
      scrapeConfigs = [
        {
          job_name = "node";
          static_configs = [
            {
              targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"];
            }
          ];
        }
      ];
    };

    services.loki = {
      enable = true;
      configuration = {
        auth_enabled = false;

        server = {
          http_listen_port = 3100;
        };

        ingester = {
          lifecycler = {
            address = "0.0.0.0";
            ring = {
              kvstore = {
                store = "inmemory";
              };
              replication_factor = 1;
            };
            final_sleep = "0s";
          };
          chunk_idle_period = "1h";
          max_chunk_age = "1h";
          chunk_target_size = 1048576;
          chunk_retain_period = "30s";
        };

        schema_config = {
          configs = [
            {
              from = "2020-10-24";
              store = "boltdb-shipper";
              object_store = "filesystem";
              schema = "v11";
              index = {
                prefix = "index_";
                period = "24h";
              };
            }
          ];
        };

        storage_config = {
          boltdb_shipper = {
            active_index_directory = "/var/lib/loki/boltdb-shipper-active";
            cache_location = "/var/lib/loki/boltdb-shipper-cache";
            cache_ttl = "24h";
          };
          filesystem = {
            directory = "/var/lib/loki/chunks";
          };
        };

        limits_config = {
          reject_old_samples = true;
          reject_old_samples_max_age = "168h";
        };


        table_manager = {
          retention_deletes_enabled = false;
          retention_period = "0s";
        };
      };
    };

    services.alloy = {
      enable = true;
    };

    # Add a simple systemd service to run the Grafana Agent (Alloy)
    systemd.services.grafana-agent = {
      description = "Grafana Alloy  - journal -> loki";
      wantedBy = ["multi-user.target"];

      serviceConfig = {
        # Ensure persistent dir for positions exists
        ExecStartPre = ''
          mkdir -p /var/lib/grafana-agent
          chown root:root /var/lib/grafana-agent
        '';

        # The grafana-agent binary provided by nixpkgs
        ExecStart = ''
          ${pkgs.grafana-alloy}/bin/grafana-alloy --config.file /etc/alloy/config.alloy
        '';

        Restart = "on-failure";
        RestartSec = "5s";
      };
    };

    # or alternatively
    #configFile = ./loki-config.yaml;
    # };

    # Enable Tailscale for remote access to Traefik dashboard and configuration
    services.tailscale.enable = true;

    networking.firewall.allowedTCPPorts = [3000];
    networking.firewall.allowedUDPPorts = [3000];
  };
}