Solved the edge case of Loki failing at first startup due to folders not initialized.

nix-system-configs/modules/system/build_machine.nix

@@ -42,6 +42,7 @@ in {
       (choose [./modules/local/hostname_username.nix ../local/hostname_username.nix])
       (choose [./modules/local/networking_local.nix ../local/networking_local.nix])
       (choose [./modules/lix-default.nix ../lix-default.nix])
+      (choose [./modules/toolsets/grafana_metric.nix ../toolsets/grafana_metric.nix])
       (choose [./modules/secrets-config/sops-build-machine.nix ../secrets-config/sops-build-machine.nix])
       ## TODO: Make this more elegant and less risky for unexpected errors happening.
     ]

nix-system-configs/modules/system/compose-songsheet.nix

@@ -38,6 +38,7 @@ in {
     # Optionally: (choose [ ./modules/toolsets/remote_building.nix ../toolsets/remote_building.nix ])
 
     ## Compose modules for Portainer service
+    (choose [./modules/toolsets/grafana_metric.nix ../toolsets/grafana_metric.nix])
     (choose [./modules/songsheet/wavelog/docker-compose.nix ../songsheet/wavelog/docker-compose.nix])
     (choose [./modules/secrets-config/sops-composesongsheet.nix ../secrets-config/sops-composesongsheet.nix])
   ];

nix-system-configs/modules/system/database.nix

@@ -34,8 +34,9 @@ in {
       (choose [./modules/desktop-manager/gnome.nix ../desktop-manager/gnome.nix])
       (choose [./modules/local/hostname_username.nix ../local/hostname_username.nix])
       (choose [./modules/local/networking_local.nix ../local/networking_local.nix])
-      (choose [./modules/bootloader/seabios-assigned-iso-at-birth.nix ../bootloader/seabios-assigned-iso-at-birth.nix])
+      # (choose [./modules/bootloader/seabios-assigned-iso-at-birth.nix ../bootloader/seabios-assigned-iso-at-birth.nix])
       (choose [./modules/lix-default.nix ../lix-default.nix])
+      (choose [./modules/toolsets/grafana_metric.nix ../toolsets/grafana_metric.nix])
       (choose [./modules/secrets-config/sops-database.nix ../secrets-config/sops-database.nix])
       (choose [./modules/system_scripts/gcloud_backup.nix ../system_scripts/gcloud_backup.nix])
       ## TODO: Make this more elegant and less risky for unexpected errors happening.

nix-system-configs/modules/system/forgejo.nix

@@ -63,6 +63,7 @@ in {
     (choose [./modules/toolsets/remote_building.nix ../toolsets/remote_building.nix])
     (choose [./modules/bootloader/seabios-assigned-proxmox-at-birth.nix ../bootloader/seabios-assigned-proxmox-at-birth.nix])
     (choose [./modules/lix-default.nix ../lix-default.nix])
+    (choose [./modules/toolsets/grafana_metric.nix ../toolsets/grafana_metric.nix])
   ];
 
   config = {

nix-system-configs/modules/system/gramethus.nix

@@ -76,6 +76,62 @@ in {
         # Prevents Grafana from phoning home
         #analytics.reporting_enabled = false;
       };
+
+      provision = {
+        enable = true;
+        # ============== Declarative Config ================
+
+        datasources.settings.datasources = [
+          # Provisioning a built-in data source
+          {
+            name = "Prometheus - Forgejo";
+            type = "prometheus";
+            url = "http://10.1.1.4:9001";
+            isDefault = false;
+            editable = true;
+          }
+
+          {
+            name = "Loki - Forgejo";
+            type = "loki";
+            url = "http://10.1.1.4:3100";
+            isDefault = false;
+            editable = true;
+          }
+
+          {
+            name = "Prometheus - Build Machine";
+            type = "prometheus";
+            url = "http://10.1.1.3:9001";
+            isDefault = false;
+            editable = true;
+          }
+
+          {
+            name = "Loki - Build Machine";
+            type = "loki";
+            url = "http://10.1.1.3:3100";
+            isDefault = false;
+            editable = true;
+          }
+
+          {
+            name = "Prometheus - Database";
+            type = "prometheus";
+            url = "http://10.1.1.251:9001";
+            isDefault = false;
+            editable = true;
+          }
+
+          {
+            name = "Loki - Database";
+            type = "loki";
+            url = "http://10.1.1.251:3100";
+            isDefault = false;
+            editable = true;
+          }
+        ];
+      };
     };
 
     services.prometheus = {
@@ -217,7 +273,5 @@ in {
 
     networking.firewall.allowedTCPPorts = [3000];
     networking.firewall.allowedUDPPorts = [3000];
-
-    # ============== Declarative Config ================
   };
 }

nix-system-configs/modules/toolsets/grafana_metric.nix

@@ -136,4 +136,16 @@
 
   networking.firewall.allowedTCPPorts = [3100 9001];
   networking.firewall.allowedUDPPorts = [3100 9001];
+
+  systemd.services."loki-create-dirs" = {
+    description = "Create Loki runtime/storage directories";
+    after = ["network.target"];
+    serviceConfig = {
+      Type = "oneshot";
+      ExecStart = ''        ${pkgs.bash}/bin/bash -c "set -eu; \
+               mkdir -p /var/loki /var/loki/compactor
+      '';
+    };
+    wantedBy = ["multi-user.target"];
+  };
 }