test: trying to get microVMs to work

modules/nixos/default.nix

@@ -54,7 +54,7 @@ in
           # glance
           jellyfin
           # logrotate
-          # mastodon
+          mastodon
           microvm
           # minecraft
           # ollamaCeres

modules/nixos/guests/mastodon/default.nix

@@ -52,7 +52,7 @@ in
           ${serviceCfg.name} = {
             enable = true;
             localDomain = host;
-            secretKeyBaseFile = "/run/secrets/pass";
+            secretKeyBaseFile = "/run/mastodon-secrets/pass";
             streamingProcesses = 7;
             trustedProxy = hostCfg.localhost.address1;
             automaticMigrations = true;
@@ -61,7 +61,7 @@ in
               name = serviceCfg.name;
               host = "/run/postgresql";
               user = serviceCfg.name;
-              passwordFile = "/run/secrets/database";
+              passwordFile = "/run/mastodon-secrets/database";
             };
             extraConfig = {
               SINGLE_USER_MODE = "true";
@@ -116,12 +116,13 @@ in
               createLocally = false;
               fromAddress = "upRootNutrition <${smtpCfg.email.address1}>";
               host = smtpCfg.hostname;
-              passwordFile = "/run/secrets/smtp";
+              passwordFile = "/run/mastodon-secrets/smtp";
               port = smtpCfg.ports.port1;
               user = smtpCfg.email.address1;
             };
           };
           caddy = {
+            enable = true;
             virtualHosts = {
               "${serviceCfg.interface.ip}" = {
                 extraConfig = ''
@@ -143,8 +144,6 @@ in
                     reverse_proxy * unix//run/mastodon-web/web.socket
                   }
 
-                  tls ${serviceCfg.ssl.cert} ${serviceCfg.ssl.key}
-
                   handle_errors {
                     root * ${pkgs.mastodon}/public
                     rewrite 500.html
@@ -156,6 +155,7 @@ in
                   header /* {
                     Strict-Transport-Security "max-age=31536000;"
                   }
+
                   header /emoji/* Cache-Control "public, max-age=31536000, immutable"
                   header /packs/* Cache-Control "public, max-age=31536000, immutable"
                   header /system/accounts/avatars/* Cache-Control "public, max-age=31536000, immutable"
@@ -181,6 +181,9 @@ in
         users.users.${serviceCfg.name}.extraGroups = [
           "postgres"
         ];
+        users.users.caddy.extraGroups = [
+          serviceCfg.name
+        ];
 
         networking.firewall.allowedTCPPorts = [
           22 # SSH
@@ -192,27 +195,43 @@ in
           5432 # Postgres
         ];
 
-        fileSystems."/tmp" = {
-          device = "tmpfs";
-          fsType = "tmpfs";
-          options = [
-            "size=4G"
-            "mode=1777"
-          ];
-        };
-
         systemd = {
           services = {
             systemd-networkd.wantedBy = [ "multi-user.target" ];
-            caddy.serviceConfig.ReadWriteDirectories = lib.mkForce [
+            copy-secrets-to-tmpfs = {
-              "/var/lib/caddy"
+              description = "Copy secrets from virtiofs to tmpfs";
-              "/run/mastodon-web"
+              wantedBy = [ "multi-user.target" ];
-            ];
+              before = [ "mastodon-init-dirs.service" ];
+              serviceConfig = {
+                Type = "oneshot";
+                RemainAfterExit = true;
+              };
+              script = ''
+                mkdir -p /run/mastodon-secrets
+                mkdir -p /run/mastodon-web
+                cp /run/secrets/pass /run/mastodon-secrets/pass
+                cp /run/secrets/smtp /run/mastodon-secrets/smtp
+                cp /run/secrets/database /run/mastodon-secrets/database
+                cp /run/secrets/redis /run/mastodon-secrets/redis
+                chown root:mastodon /run/mastodon-secrets/*
+                chmod 0640 /run/mastodon-secrets/*
+                chown mastodon:mastodon /run/mastodon-web
+                chmod 0755 /run/mastodon-web
+              '';
+            };
+            caddy = {
+              after = [ "copy-secrets-to-tmpfs.service" ];
+              requires = [ "copy-secrets-to-tmpfs.service" ];
+              serviceConfig.ReadWriteDirectories = lib.mkForce [
+                "/var/lib/caddy"
+                "/run/mastodon-web"
+              ];
+            };
           };
           network = {
             enable = true;
             networks."20-lan" = {
-              matchConfig.Name = "enp0s5";
+              matchConfig.Name = "enp0s6";
               addresses = [ { Address = "${serviceCfg.interface.ip}/24"; } ];
               routes = [
                 {
@@ -229,12 +248,13 @@ in
 
           tmpfiles.rules = [
             "Z ${serviceCfg.varPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
+            "Z /var/lib/postgresql 0755 postgres postgres -"
           ];
         };
 
         microvm = {
-          vcpu = 2;
+          vcpu = 4;
-          mem = 3072;
+          mem = 8192;
           hypervisor = "qemu";
           interfaces = [
             {
@@ -265,9 +285,15 @@ in
             {
               mountPoint = "/var/lib/${serviceCfg.name}";
               proto = "virtiofs";
-              source = "${serviceCfg.mntPaths.path0}";
+              source = "${serviceCfg.mntPaths.path0}/data";
               tag = "${serviceCfg.name}_data";
             }
+            {
+              mountPoint = "/var/lib/postgresql";
+              proto = "virtiofs";
+              source = "${serviceCfg.mntPaths.path0}/database";
+              tag = "${serviceCfg.name}_database";
+            }
             {
               mountPoint = "/run/secrets";
               proto = "virtiofs";
@@ -301,11 +327,14 @@ in
 
   systemd.tmpfiles.rules = [
     "d ${serviceCfg.mntPaths.path0} 0751 microvm wheel - -"
+    "d ${serviceCfg.mntPaths.path0}/data 0751 microvm wheel - -"
+    "d ${serviceCfg.mntPaths.path0}/database 0751 microvm wheel - -"
+
   ];
 
   services.caddy.virtualHosts."${host}" = {
     extraConfig = ''
-      reverse_proxy ${serviceCfg.interface.ip}:80
+      reverse_proxy http://${serviceCfg.interface.ip}:80
 
       tls ${serviceCfg.ssl.cert} ${serviceCfg.ssl.key}
 

systems/ceres/config/sops.nix

@@ -2,7 +2,7 @@
   sops = {
     secrets = {
       "network/server" = {
-        path = "/opt/secrets/server";
+        path = "/var/lib/secrets/server";
         owner = "root";
         mode = "600";
       };