diff --git a/dry_run.zsh b/dry_run.zsh index db5f6c5..9920b80 100755 --- a/dry_run.zsh +++ b/dry_run.zsh @@ -15,6 +15,7 @@ configs=( "nixos-teamspeak" "nixos-traefik" "nixos-build-machine" + "nixos-logs" ) if [[ -n "$1" ]]; then diff --git a/flake.nix b/flake.nix index ad1ebd7..c43d220 100644 --- a/flake.nix +++ b/flake.nix @@ -52,6 +52,11 @@ inherit system; modules = [./nix-system-configs/modules/system/build_machine.nix]; }; + + "nixos-logs" = nixpkgs.lib.nixosSystem { + inherit system; + modules = [./nix-system-configs/modules/system/gramethus.nix]; + }; }; }; } diff --git a/nix-system-configs/modules/system/gramethus.nix b/nix-system-configs/modules/system/gramethus.nix new file mode 100644 index 0000000..2a08dae --- /dev/null +++ b/nix-system-configs/modules/system/gramethus.nix @@ -0,0 +1,223 @@ +{ + config, + pkgs, + lib, + ... +}: let + choose = paths: lib.findFirst builtins.pathExists null paths; + + # Dummy filesystem config for dry-run evaluation + dummyFileSystems = { + fileSystems."/" = { + device = "/dev/disk/by-label/nixos"; + fsType = "ext4"; + }; + }; +in { + options.local = { + hostname = lib.mkOption { + type = lib.types.str; + default = "nixos-default"; + description = "System hostname"; + }; + username = lib.mkOption { + type = lib.types.str; + default = "user"; + description = "Primary user username"; + }; + userDescription = lib.mkOption { + type = lib.types.str; + default = "NixOS User"; + description = "Primary user description"; + }; + address = lib.mkOption { + type = lib.types.str; + default = "10.1.1.100"; + description = "Static IP address"; + }; + }; + + imports = + lib.filter (x: x != null) [ + (choose [./modules/local/hostname_username.nix ../local/hostname_username.nix]) + (choose [./modules/local/networking_local.nix ../local/networking_local.nix]) + (choose [./modules/bootloader/seabios-assigned-iso-at-birth.nix ../bootloader/seabios-assigned-iso-at-birth.nix]) + (choose [./modules/lix-default.nix ../lix-default.nix]) + (choose [./modules/secrets-config/sops-the-blank-system.nix ../secrets-config/sops-the-blank-system.nix]) + ] + # TODO: Make this more clean, migrate over to seabios-assigned-iso-at-birth.nix? + #++ lib.optional (builtins.pathExists ./hardware-configuration.nix) ./hardware-configuration.nix + #++ lib.optional (!builtins.pathExists ./hardware-configuration.nix) dummyFileSystems + ; + config = { + local.hostname = "nixos-grametheus"; + local.username = "prglogs"; + local.userDescription = "NixOS PRG Grafana Prometheus Service"; + local.address = "10.1.1.10"; + + system.stateVersion = "25.11"; + + services.grafana = { + enable = true; + settings = { + server = { + http_addr = "10.1.1.10"; + http_port = 3000; + #enforce_domain = true; + enable_gzip = true; + domain = "grafana.prg-radio.org"; + + # Alternatively, if you want to serve Grafana from a subpath: + # domain = "your.domain"; + # root_url = "https://your.domain/grafana/"; + # serve_from_sub_path = true; + }; + + # Prevents Grafana from phoning home + #analytics.reporting_enabled = false; + }; + }; + + services.prometheus = { + enable = true; + port = 9001; + + exporters.node = { + enabledCollectors = [ + "ethtool" + "softirqs" + "systemd" + "tcpstat" + ]; + enable = true; + port = 9002; + }; + globalConfig.scrape_interval = "10s"; # "1m" + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [ + { + targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"]; + } + ]; + } + ]; + }; + + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + + server = { + http_listen_port = 3100; + }; + + ingester = { + lifecycler = { + address = "0.0.0.0"; + ring = { + kvstore = { + store = "inmemory"; + }; + replication_factor = 1; + }; + final_sleep = "0s"; + }; + chunk_idle_period = "1h"; + max_chunk_age = "1h"; + chunk_target_size = 1048576; + chunk_retain_period = "30s"; + }; + + schema_config = { + configs = [ + { + from = "2020-10-24"; + store = "boltdb-shipper"; + object_store = "filesystem"; + schema = "v11"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; + + storage_config = { + boltdb_shipper = { + active_index_directory = "/var/lib/loki/boltdb-shipper-active"; + cache_location = "/var/lib/loki/boltdb-shipper-cache"; + cache_ttl = "24h"; + }; + filesystem = { + directory = "/var/lib/loki/chunks"; + }; + }; + + limits_config = { + reject_old_samples = true; + reject_old_samples_max_age = "168h"; + # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used. + allow_structured_metadata = false; + }; + + table_manager = { + retention_deletes_enabled = false; + retention_period = "0s"; + }; + }; + }; + + services.alloy = { + enable = true; + }; + + # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md + environment.etc."alloy/config.alloy" = { + text = '' + discovery.relabel "journal" { + targets = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + } + + loki.source.journal "journal" { + max_age = "12h0m0s" + relabel_rules = discovery.relabel.journal.rules + forward_to = [loki.write.default.receiver] + labels = { + host = "chrysalis", + job = "systemd-journal", + } + } + + loki.write "default" { + endpoint { + url = "http://127.0.0.1:3100/loki/api/v1/push" + } + external_labels = {} + } + ''; + group = "root"; + mode = "0644"; + }; + + # or alternatively + #configFile = ./loki-config.yaml; + # }; + + # Enable Tailscale for remote access to Traefik dashboard and configuration + services.tailscale.enable = true; + + networking.firewall.allowedTCPPorts = [3000]; + networking.firewall.allowedUDPPorts = [3000]; + + # ============== Declarative Config ================ + }; +} diff --git a/nix-system-configs/modules/system/traefik.nix b/nix-system-configs/modules/system/traefik.nix index 0d7d0d0..a458f3a 100644 --- a/nix-system-configs/modules/system/traefik.nix +++ b/nix-system-configs/modules/system/traefik.nix @@ -58,7 +58,7 @@ in { TARGET = " "; # Configure redirect domains - ALL domains that should be able to redirect back after challenge - REDIRECT_DOMAINS = "prg-radio.org,anubis.prg-radio.org,git.prg-radio.org,lgbtq.prg-radio.org,wavelog.prg-radio.org,partdb.prg-radio.org"; + REDIRECT_DOMAINS = "prg-radio.org,anubis.prg-radio.org,git.prg-radio.org,lgbtq.prg-radio.org,wavelog.prg-radio.org,partdb.prg-radio.org,grafana.prg-radio.org"; # Public URL for Anubis PUBLIC_URL = "https://anubis.prg-radio.org"; @@ -78,7 +78,7 @@ in { # Prevents infinite loop of doom: # See: https://github.com/TecharoHQ/anubis/issues/970 - JWT_RESTRICTION_HEADER="CF-Connecting-IP"; + JWT_RESTRICTION_HEADER = "CF-Connecting-IP"; # Metrics on separate port METRICS_BIND_NETWORK = "tcp"; @@ -90,14 +90,142 @@ in { COOKIE_SAME_SITE = "None"; # Only if Secure=true and you need cross-site }; }; - }; }; + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + + server = { + http_listen_port = 3100; + }; + + ingester = { + lifecycler = { + address = "0.0.0.0"; + ring = { + kvstore = { + store = "inmemory"; + }; + replication_factor = 1; + }; + final_sleep = "0s"; + }; + chunk_idle_period = "1h"; + max_chunk_age = "1h"; + chunk_target_size = 1048576; + chunk_retain_period = "30s"; + }; + + schema_config = { + configs = [ + { + from = "2020-10-24"; + store = "boltdb-shipper"; + object_store = "filesystem"; + schema = "v11"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; + + storage_config = { + boltdb_shipper = { + active_index_directory = "/var/lib/loki/boltdb-shipper-active"; + cache_location = "/var/lib/loki/boltdb-shipper-cache"; + cache_ttl = "24h"; + }; + filesystem = { + directory = "/var/lib/loki/chunks"; + }; + }; + + limits_config = { + reject_old_samples = true; + reject_old_samples_max_age = "168h"; + # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used. + allow_structured_metadata = false; + }; + + table_manager = { + retention_deletes_enabled = false; + retention_period = "0s"; + }; + }; + }; + + services.alloy = { + enable = true; + }; + + services.prometheus = { + enable = true; + port = 9001; + + exporters.node = { + enabledCollectors = [ + "ethtool" + "softirqs" + "systemd" + "tcpstat" + ]; + enable = true; + port = 9002; + }; + globalConfig.scrape_interval = "5s"; # "1m" + scrapeConfigs = [ + { + job_name = "traefik"; + static_configs = [ + { + targets = ["localhost:8080"]; + } + ]; + } + ]; + }; + + # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md + environment.etc."alloy/config.alloy" = { + text = '' + discovery.relabel "metrics_integrations_integrations_traefik" { + targets = [{ + __address__ = "localhost:8080", + }] + + rule { + target_label = "instance" + replacement = constants.hostname + } + } + + prometheus.scrape "metrics_integrations_integrations_traefik" { + targets = discovery.relabel.metrics_integrations_integrations_traefik.output + job_name = "integrations/traefik" + } + ''; + group = "root"; + mode = "0644"; + }; + services.traefik = { enable = true; group = "acme"; staticConfigOptions = { + metrics = { + prometheus = { + addEntryPointsLabels = true; + addRoutersLabels = true; + addServicesLabels = true; + buckets = [0.1 0.3 1.2 5.0]; + }; + }; + entryPoints = { web = { # Bind on IPv6 as well as IPv4 by using [::]:80 @@ -174,9 +302,9 @@ in { anubis = { rule = "Host(`anubis.prg-radio.org`)"; service = "anubis"; - entryPoints = [ "websecure" ]; + entryPoints = ["websecure"]; tls = { - certresolver = "acme"; + certresolver = "acme"; }; }; @@ -184,16 +312,16 @@ in { forgejo = { rule = "Host(`git.prg-radio.org`)"; service = "forgejo"; - entryPoints = [ "websecure" ]; + entryPoints = ["websecure"]; tls = {}; - middlewares = [ "anubisForwardAuth" ]; + middlewares = ["anubisForwardAuth"]; }; # Matrix HTTP router for client requests (Element etc.) matrix = { rule = "Host(`lgbtq.prg-radio.org`)"; service = "matrix"; - entryPoints = [ "websecure" ]; + entryPoints = ["websecure"]; tls = {}; }; @@ -201,7 +329,7 @@ in { wavelog = { rule = "Host(`wavelog.prg-radio.org`)"; service = "wavelog"; - entryPoints = [ "websecure" ]; + entryPoints = ["websecure"]; tls = {}; }; @@ -209,9 +337,18 @@ in { partdb = { rule = "Host(`partdb.prg-radio.org`)"; service = "partdb"; - entryPoints = [ "websecure" ]; + entryPoints = ["websecure"]; tls = {}; - middlewares = [ "anubisForwardAuth" ]; + middlewares = ["anubisForwardAuth"]; + }; + + # Protected service: Grafana + grafana = { + rule = "Host(`grafana.prg-radio.org`)"; + service = "grafana"; + entryPoints = ["websecure"]; + tls = {}; + middlewares = ["anubisForwardAuth"]; }; }; @@ -244,6 +381,12 @@ in { {url = "http://10.1.1.249:8087";} ]; }; + + grafana.loadBalancer = { + servers = [ + {url = "http://10.1.1.10:3000";} + ]; + }; }; # TCP routing for TeamSpeak @@ -339,9 +482,9 @@ in { wants = ["acme-finished-prg-radio.org.target"]; }; - networking.firewall.allowedTCPPorts = [80 443 10022 30033 8448]; - networking.firewall.allowedUDPPorts = [80 443 9987]; + networking.firewall.allowedTCPPorts = [80 443 10022 30033 8448 9001 3001]; + networking.firewall.allowedUDPPorts = [80 443 9987 9001 3001]; system.stateVersion = "25.11"; }; -} \ No newline at end of file +} diff --git a/nix-system-configs/modules/toolsets/grafana_metric.nix b/nix-system-configs/modules/toolsets/grafana_metric.nix new file mode 100644 index 0000000..5cc95f9 --- /dev/null +++ b/nix-system-configs/modules/toolsets/grafana_metric.nix @@ -0,0 +1,139 @@ +{ + config, + pkgs, + lib, + ... +}: { + services.prometheus = { + enable = true; + port = 9001; + + exporters.node = { + enabledCollectors = [ + "ethtool" + "softirqs" + "systemd" + "tcpstat" + ]; + enable = true; + port = 9002; + }; + globalConfig.scrape_interval = "10s"; # "1m" + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [ + { + targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"]; + } + ]; + } + ]; + }; + + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + + server = { + http_listen_port = 3100; + }; + + ingester = { + lifecycler = { + address = "0.0.0.0"; + ring = { + kvstore = { + store = "inmemory"; + }; + replication_factor = 1; + }; + final_sleep = "0s"; + }; + chunk_idle_period = "1h"; + max_chunk_age = "1h"; + chunk_target_size = 1048576; + chunk_retain_period = "30s"; + }; + + schema_config = { + configs = [ + { + from = "2020-10-24"; + store = "boltdb-shipper"; + object_store = "filesystem"; + schema = "v11"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; + + storage_config = { + boltdb_shipper = { + active_index_directory = "/var/lib/loki/boltdb-shipper-active"; + cache_location = "/var/lib/loki/boltdb-shipper-cache"; + cache_ttl = "24h"; + }; + filesystem = { + directory = "/var/lib/loki/chunks"; + }; + }; + + limits_config = { + reject_old_samples = true; + reject_old_samples_max_age = "168h"; + # Disable structured metadata/OTLP native ingestion until schema is upgraded to v13+ and tsdb index is used. + allow_structured_metadata = false; + }; + + table_manager = { + retention_deletes_enabled = false; + retention_period = "0s"; + }; + }; + }; + + services.alloy = { + enable = true; + }; + + # See: https://github.com/grafana/alloy/blob/bbe11b0f6a3bd3391108b2d27122c654c535dd8d/docs/sources/set-up/migrate/from-promtail.md + environment.etc."alloy/config.alloy" = { + text = '' + discovery.relabel "journal" { + targets = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + } + + loki.source.journal "journal" { + max_age = "12h0m0s" + relabel_rules = discovery.relabel.journal.rules + forward_to = [loki.write.default.receiver] + labels = { + host = "chrysalis", + job = "systemd-journal", + } + } + + loki.write "default" { + endpoint { + url = "http://127.0.0.1:3100/loki/api/v1/push" + } + external_labels = {} + } + ''; + group = "root"; + mode = "0644"; + }; + + networking.firewall.allowedTCPPorts = [3100 9001]; + networking.firewall.allowedUDPPorts = [3100 9001]; +}