Implented Google Cloud Backups of the databases, and also encrypying them before being pushed to cloud. Let us hope that I didn't heck it up.

.sops.yaml

@@ -2,6 +2,7 @@ keys:
   - &admin_christine age1746rvsvsc3snxfl7cndm222wd5kck4aqj3x7nednlegq0gdjhfcqx0qv7m
   - &server_songsheet age1la8yjuc2ws9pyx70rc83jd2084v5e08v8fvh6muvfzrl2ulp8fms6frs86
   - &server_traefik age1rdcs8y4fjfyagwt2q9599ax329thceersh6dg2f0p6nsghm5xufq00qu0p
+  - &server_database age1k9ddvzypz986a7dt403ja6evql2agz0gehll79mx64zceteya38smxph8m
 creation_rules:
   - path_regex: nix-system-configs/secrets/songsheet/[^/]+\.(yaml|json|env|ini)$
     key_groups:
@@ -16,5 +17,5 @@ creation_rules:
   - path_regex: nix-system-configs/secrets/database/[^/]+\.(yaml|json|env|ini)$
     key_groups:
       - age:
-          - *admin_christine
+        - *admin_christine
-          - *server_traefik
+        - *server_database

nix-system-configs/modules/secrets-config/sops-database.nix

@@ -25,4 +25,19 @@
   sops.age.keyFile = "/var/lib/sops-nix/key.txt";
   # This will generate a new key if the key specified above does not exist
   sops.age.generateKey = true;
+  # Secrets key stuff
+  sops.secrets."backup_gpg_key" = {
+    format = "yaml";
+    sopsFile = ../../secrets/database/secrets.yaml;
+    owner = "root";
+    mode = "0400";
+  };
+
+  # Declare the GCS service account secret
+  sops.secrets."gcloud_bucket" = {
+    format = "json";
+    sopsFile = ../../secrets/database/gcloud_bucket.json;
+    owner = "root";
+    mode = "0400";
+  };
 }

nix-system-configs/modules/system/database.nix

@@ -33,6 +33,7 @@
     ./modules/local/networking_local.nix
     ./modules/lix-default.nix
     ./modules/secrets-config/sops-database.nix
+    ./modules/database/gcs-backups.nix
     ./hardware-configuration.nix
   ];
 

nix-system-configs/modules/system_scripts/backup_mariadb.zsh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+TIMESTAMP=$(date +%Y%m%d%H%M%S)
+BACKUP_DIR=$(mktemp -d)
+trap 'rm -rf "$BACKUP_DIR"' EXIT
+
+gpg --batch --import "${GPG_KEY_FILE}"
+
+DATABASES=$(mysql -u root -e "SHOW DATABASES;" | grep -Ev "^(Database|information_schema|performance_schema|mysql|sys)$")
+
+for DB in $DATABASES; do
+  echo "Backing up MariaDB database: $DB"
+
+  FILENAME="mariadb_${DB}_${TIMESTAMP}.sql.gz.gpg"
+  if mysqldump -u root "$DB" | gzip | gpg --batch --encrypt --recipient "${GPG_RECIPIENT}" > "$BACKUP_DIR/$FILENAME"; then
+    gsutil cp "$BACKUP_DIR/$FILENAME" "gs://${GCS_BUCKET}/mariadb/$FILENAME"
+    echo "Successfully uploaded encrypted $FILENAME"
+  else
+    echo "Failed to backup $DB" >&2
+    exit 1
+  fi
+done

nix-system-configs/modules/system_scripts/backup_postgresql.zsh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+TIMESTAMP=$(date +%Y%m%d%H%M%S)
+BACKUP_DIR=$(mktemp -d)
+trap 'rm -rf "$BACKUP_DIR"' EXIT
+
+# Import GPG key for encryption
+gpg --batch --import "${GPG_KEY_FILE}"
+
+DATABASES=$(psql -U postgres -t -c "SELECT datname FROM pg_database WHERE datistemplate = false AND datname != 'postgres';" | grep -v '^$')
+
+for DB in $DATABASES; do
+  DB=$(echo "$DB" | xargs)
+  echo "Backing up PostgreSQL database: $DB"
+  FILENAME="pgsql_${DB}_${TIMESTAMP}.sql.gz.gpg"
+  if pg_dump -U postgres -d "$DB" | gzip | gpg --batch --encrypt --recipient "${GPG_RECIPIENT}" > "$BACKUP_DIR/$FILENAME"; then
+    gsutil cp "$BACKUP_DIR/$FILENAME" "gs://${GCS_BUCKET}/postgresql/$FILENAME"
+    echo "Successfully uploaded encrypted $FILENAME"
+  else
+    echo "Failed to backup $DB" >&2
+    exit 1
+  fi
+done

nix-system-configs/modules/system_scripts/gcloud_backup.nix

@@ -0,0 +1,122 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  gcsBucket = "polyteknisk-db-backups-bucket";
+  gpgRecipient = "dtu.prg@dtu.dk";
+
+  # Embed the public key directly (not sensitive)
+  gpgPublicKey = pkgs.writeText "backup-public-key.asc" ''
+    -----BEGIN PGP PUBLIC KEY BLOCK-----
+
+    mQINBGmHsFcBEADuiQeZSU2KMBT3+PAECEi4qW87WbMpeMIJlN7b5dNvSIVgaOie
+    n/KNDcZiKL5WkxIoxcEy1k64EQ8oY+xdAgUItZAL4zoB2bBY2rHmbhaRx36rYOdZ
+    ECI1VAe+yQ6gMSCZ26HxTSQebwocXC7LYj2qkn32TwQfNwl/93/ZSb64Np+qt8zL
+    ym0yl6ruz+du2qd9eAOJSVKAiRsvvW6LxC/+jBqdgBpYXhvjtqC8wtS6hW26jing
+    BauktYy3hjWy14JPCjSMIclIx7zqkW3+gu7lvGu9osKXwdjMWC1R+j1UvGufNczK
+    O05RR9EVrfbrv2P0+JoxQhRhEN3TXjRmx98YwPOHzD6rG2u2mIM666Huxrl8xKhn
+    cxs/p0lcbUhrmiwr3rn33r0v91s5KFhBxSNQs3FnDfET9zGx+YPwoVDpR61EelfD
+    lCowWHmsjXXjcvOBGwA5quYaep7idgM9gcmMVh6OKgHo6NlgBAeQQOSsF9u832OH
+    TjttMUFw0+NeaVO1hWFQgQvBNIk1b7qp77sFwRrrXsE1orC0ii4yODQ+OV7C2A8G
+    2Non0yHWST4scd8WbiEOv3SLxyGlV+hER/HtHeQ7Ke8vvMUW5ZObv4qy17RHNJZS
+    lQY4/cBODV04sLz2GGEChpeZ8w2ebTHxp2LhhedPgYLyi5Fr6OMcJe3WvwARAQAB
+    tGtQUkcgQmFja3VwIChUaGlzIGlzIGEgR1BHIEtleSB0byBjbG91ZCBiYWNrdXAg
+    ZGF0YWJhc2VzIGJlZm9yZSBiZWluZyBwdXNoZWQgdG8gdGhlIGNsb3VkLikgPGR0
+    dS5wcmdAZHR1LmRrPokCUQQTAQgAOxYhBI+Qr5IHE1lTHC/nNY/ZDvQidP7iBQJp
+    h7BXAhsDBQsJCAcCAiICBhUKCQgLAgQWAgMBAh4HAheAAAoJEI/ZDvQidP7izk0Q
+    AK6m5jkKU8sGsKdTqmE8cd6cLtFsbTnk6abi6iv0ZZ1wsWNie/fdy8RGuaQJOvpm
+    FDxGaMaSYoMsdG/ONNRH/kBWybHba51c19Rr5ePKoL0aHHMnYuzT1xzPyQ/a/Ckh
+    8WyIYflHRO0DU8sonnKlDZiwaq/bfu3FL03IsnPVbQOpZ0o8c9HIxDG0efgvO5li
+    v+ajiDMld5iwvSBz9+9yDDiSNYPRipNlhaFDaoT9pM6s7ETmRCYQ75T7z0FMaugt
+    iTJRz0emyj/canJ2/JrkVBGNw8Kb9JG8Nd6ix2oZagcv96niRKQoqzWyiWizNWnq
+    /wQ7QHqMxXGymNseF8Ig/a/0oJs+mRYYCBNj2hkF2COQdHqSr+NduSJfaZGCdHOe
+    kTeWtit9iWzeXnA6FjQX3SXynL89Nv9uJlfYZQX5Kai1DIxvPGvIpI3M1vVjGRfG
+    0/WNwoCOmLRhGndaW9iqk7Xk7+k43q4CAsEsKhNQnAH7TzHOuOUmM7pSi3mK4Eby
+    6tmiz0nJRdXwh0Fgl3VUZfU6aQqD5cdPDP4EAJpUuR1LCHMQLkDEL61gibBvuPkv
+    nsuqKuG2avFU87CCxZjusWtvp11Pv3JA2gmVPETWp3Gw0Yi+sIgLFKI4HOyb2UdB
+    STooGKvwHVfGm+AKovitDFT0eHlcUCkJFP6U+TRCYKNMuQINBGmHsFcBEACkbyGK
+    9T3+FJydvYAdCkkdohTy4yyOuOC+9P3Fv3LN2jM+CqdW8LPKZKgU4upemae3hANQ
+    345fTebYqvrCyaHbdSV64q9leV0M/NwAiDOpWYHUVuM2vKciBTalx+d9TgnW5gLk
+    QuG+G+gD43GLr9wj/Ewuyj0bGx16jzFk0iD9ZcCEvBjE3Fymd3UFQmFMpsMPZ+CB
+    KgnwksD0SmUFA1budhB+okFGfcHympMwebKCR30Kb1/MV9Y3GgPwjzOeL3vckhuo
+    0bYKh6FLHv83DRnVDTBA2LgT9AprjHIZAOgpVyWWFz6WU5oSIGnHCrf9RSLor1BR
+    QwRQsoqWgea4qSDfnyCzZcmYf4H8aV/QOl5GDSSQ4XWFBRGZ/gke3AiRKTRCFvgg
+    0iDL2m4oNuHlnM/J2sg25mBMItutPPK+UFQWDVvaVXe64/yDaKOLFN+9k5GGneIG
+    rh0ucCY3OOUZPotC94YbTWSZGmxwouFVvfHkuxN9DtL2w9au/60i+Z0/cbMfUgqc
+    URsgxnGTb7OncCWj3WNqLVoxF4l8lFEKQXx/EJblasx68vV318q7AsTCWOK+c3m6
+    K50yXfagyqLbcXKwmSpYz/NJ6ozyVmd1LEULNpP2TPrCE5n/iCG284IRkESRVkgC
+    JdXOL8jdTcUHUAvZ0MRonXydRhWOvJlljoxlgQARAQABiQI2BBgBCAAgFiEEj5Cv
+    kgcTWVMcL+c1j9kO9CJ0/uIFAmmHsFcCGwwACgkQj9kO9CJ0/uLQzg/9F1FekOup
+    NOSwqfCWWZCURe0onigkGlRmVzxbefkiFWyBPoVIpyPWEOfKBDlSSeg7caeYyXK0
+    3s9uLFJ/qiusIFhgts/Y4nmbW3E6CfuOUhwukKJqWbv2HTBDzypUgiNiyYWSitU0
+    Xt1I3P3M8rAp1X/WJ/jZ5pTGqc9Rh+zvQIuYU3+MWOVds6nfPpUFv5EDsa88puLS
+    ti5O7/dqV6TEp9qpDofKkIiUlWbDZQ1te2DmasgskGwVMLNZ5rxwzSxrZKwHAhQ4
+    7FuvISIK5QOy2dEjVdjgB9eD+RBvDbsxB/i5bIn5l2TPIaKq0SzcG3N2msfUUGuF
+    BGeRgAnlPgLH+2rOZh6KH8FlRUHH9pxkiSo87qXmgxkA1XNeZp0wV8yOFWiPvW22
+    kgTmmOIftXIfMWFlM6+fp+WDO3Fhh3XvSvjeBHJRaSp3e9dRqm5U8qtoGfqgJklE
+    6TJBj6xOqh+i3CC/4EymjvO5+1X9CUsR+TsKvT0t7c0DW78S+Ly+gKAUPgNv0ifs
+    3GU403nHnHD/Esl9r+xLdMPdU4x37uL48dByxHZ1ThR11u5QZBSVSToRQFca4/nL
+    swC2tsvzP503rVKFehdavT7wxYER03Gz6/y5JueHuBN5RgP6H6PLBu32FjDQPLK2
+    DFYqkeE28Uzz52DpHGvbw4cT08w17WWrf6A=
+    =jpAS
+    -----END PGP PUBLIC KEY BLOCK-----
+  '';
+
+  postgresBackupScript = pkgs.writeShellScript "backup-postgresql-wrapper" ''
+    export GOOGLE_APPLICATION_CREDENTIALS="${config.sops.secrets.gcloud_bucket.path}"
+    export GCS_BUCKET="${gcsBucket}"
+    export GPG_RECIPIENT="${gpgRecipient}"
+    export GPG_PUBLIC_KEY="${gpgPublicKey}"
+    export PATH="${lib.makeBinPath [pkgs.postgresql pkgs.gzip pkgs.google-cloud-sdk pkgs.gnupg]}:$PATH"
+
+    exec ${./backup_postgresql.sh}
+  '';
+
+  mariadbBackupScript = pkgs.writeShellScript "backup-mariadb-wrapper" ''
+    export GOOGLE_APPLICATION_CREDENTIALS="${config.sops.secrets.gcloud_bucket.path}"
+    export GCS_BUCKET="${gcsBucket}"
+    export GPG_RECIPIENT="${gpgRecipient}"
+    export GPG_PUBLIC_KEY="${gpgPublicKey}"
+    export PATH="${lib.makeBinPath [pkgs.mariadb pkgs.gzip pkgs.google-cloud-sdk pkgs.gnupg]}:$PATH"
+
+    exec ${./backup_mariadb.sh}
+  '';
+in {
+  systemd.services.backup-postgresql = {
+    description = "Backup PostgresSQL databases to GCS";
+    serviceConfig = {
+      Type = "oneshot";
+      User = "postgres";
+      ExecStart = "${postgresBackupScript}";
+    };
+  };
+
+  systemd.timers.backup-postgresql = {
+    description = "Timer for PostgresSQL backups";
+    wantedBy = ["timers.target"];
+    timerConfig = {
+      OnCalendar = "Mon,Thu,Sun 03:00";
+      Persistent = true;
+    };
+  };
+
+  systemd.services.backup-mariadb = {
+    description = "Backup MariaDB databases to GCS";
+    serviceConfig = {
+      Type = "oneshot";
+      User = "root";
+      ExecStart = "${mariadbBackupScript}";
+    };
+  };
+
+  systemd.timers.backup-mariadb = {
+    description = "Timer for MariaDB backups";
+    wantedBy = ["timers.target"];
+    timerConfig = {
+      OnCalendar = "Mon,Thu,Sun 04:00";
+      Persistent = true;
+    };
+  };
+}

nix-system-configs/secrets/database/gcloud_bucket.json

@@ -14,15 +14,15 @@
 		"age": [
 			{
 				"recipient": "age1746rvsvsc3snxfl7cndm222wd5kck4aqj3x7nednlegq0gdjhfcqx0qv7m",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIeTFMQlBOS1kyNjJkYito\nZG9XQ3NJSmJxNk1pZDF2T3FoOUU5SEhqYmw0CmN4VnE5cWR3Tm1kdkZnWEZJcHha\nejBNODZkOXFnckRFdTVwdFJwTUVWaDgKLS0tIGRCY2VaS2RMcjQ4clZRbG52ZGdI\nQXdZZ3hBcktmMUlhNDV4TGVaT2c0UEkKH9e+rTKrRt9JqYG+RkFrlcaNXd8zn+0/\no65SOKlwMC0VAAb7rDDU0xGmahW2/bWErW2qJ/88dvDuqdX2sD28Pg==\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvZFJFQW90YkhvN3BjSWdW\nUTB2a1pmSWFNZEthRTlJYTFkZDZ2c1VvZG1FCmxYcEZiZ2dpMjdzRVhxMDhhZ2Mz\nSEYwOXlMOTh0U2REdmZKQ0pxU3NNRHcKLS0tIHFlQXErSk9UOXRHdjgvLzFvOHRh\nZ1AwV3dCd1JFRkxSQ0RjS2E0aHUwZlEKcFgfkNX5ieo+1ao1F+bi8KiwdvvvIKC7\nVddjbENyi4XAtor0JLTeUJEwMMIu/fYEbDsejpSkkR1/Yx5AUAjpcA==\n-----END AGE ENCRYPTED FILE-----\n"
 			},
 			{
-				"recipient": "age1rdcs8y4fjfyagwt2q9599ax329thceersh6dg2f0p6nsghm5xufq00qu0p",
+				"recipient": "age1k9ddvzypz986a7dt403ja6evql2agz0gehll79mx64zceteya38smxph8m",
-				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5ZTI4dUQxdExvTjA1UDJj\nMUtFZ1F4SzZzVGhXTjJQYkRQaVlyUEZsaTFVCjdtc01Ydk9lMWd2cnVrbTAzeERN\nbG1XcTYzU1RPRXRYcFlzK3RJQzJUV2MKLS0tIEJETzZKaHZENTdrbUVTYWIxNmhB\ndWhvNEJpNVQvcG9lam1lbkV1dGxpMDAK9OspZrLOshe7JLROJvJ9dkzejkSRixyJ\nzD0IbFv3N+HIC3DeStDzCUnRdLmrM/q4HOYCPNCmAtT9jvOrD96ejw==\n-----END AGE ENCRYPTED FILE-----\n"
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhM1dsczdSTVQ0b1FPampW\neTJwZDBkQTNiZk9TNFJuaC9uTFkrSk5YTEhZCnRQbWtab0oxb1Jtay9CeDlHOWlQ\naWVsZ1JEMHluQmxETUpzQkJ6RG4yYXMKLS0tIG04M3NwUVN0MmE3RUxXMWtGTC9H\nRG1naXF1bjlHeUlLRG5ibVhuWWk2MTgKFlN/qZhfDln5fLdsrbODDfZWZEM1bLkd\nDgWwNwl6VElnLBvLcRq7vMovOBrlboGgdOt1L75himEcE1xaG07I4A==\n-----END AGE ENCRYPTED FILE-----\n"
 			}
 		],
-		"lastmodified": "2026-02-07T20:01:07Z",
+		"lastmodified": "2026-02-07T21:00:49Z",
-		"mac": "ENC[AES256_GCM,data:fjKrLuYFXZhHa2KwyWTmDiEyZInyJluI4QjevW2liWKyv7rtJbwzxlzUEb4hUG3pwFFOWeeNwAvIg+la+0Y+sERIEM5P5j60buia2cQZFTK6WvNwLsNN+/sDptznGfrUsI3GJOBtIQgdCCBrrDwj7YARP+T6kk0wh7iw6LgfuWY=,iv:Psy2bv2w9J4OXNH8MESCZD1zSCWDRKlmaOijgyKyorc=,tag:QpiPUT9WMzQQLUVTwtfWjw==,type:str]",
+		"mac": "ENC[AES256_GCM,data:9pZ6xXwUkOhNrfwz06+xJDWa7WT45p8tUERvfU3xIgKsgrGiyJRHZ2sewDE2kujrDz6qTwNL1Jo8d+sY3cmXJfSitxvwzLP9zKrV3cnpd1p/M7+RxZuz97k7XHTk97II/mlctYFg0X4oi9frbM4NFOaRsHfabfvAxOV4AMUo5Uk=,iv:HWxmk9Xda32/T6e+yECv+N6dGg/XGOKogDoW0cETId4=,tag:vkYFqE/tywo+Qv7sVe0AEg==,type:str]",
 		"unencrypted_suffix": "_unencrypted",
 		"version": "3.11.0"
 	}