From 9f6f5cda5e5308b6a0c774c44b782978cdd92c98 Mon Sep 17 00:00:00 2001
From: Nick <nickjhiebert@proton.me>
Date: Thu, 6 Nov 2025 22:20:14 -0600
Subject: [PATCH] test: vaultwarden microVM

---
 modules/config/default.nix                    |   2 +-
 modules/nixos/default.nix                     |   4 +-
 .../nixos/services/vaultwarden/default.nix    | 363 +++++++++++-------
 systems/ceres/config/bridge.nix               |  66 ----
 systems/ceres/config/networking.nix           |  18 -
 5 files changed, 219 insertions(+), 234 deletions(-)
 delete mode 100755 systems/ceres/config/bridge.nix

diff --git a/modules/config/default.nix b/modules/config/default.nix
index 5dca88b..4311455 100755
--- a/modules/config/default.nix
+++ b/modules/config/default.nix
@@ -244,7 +244,7 @@ in
 
           ceresStorageDriveName = "NAS1";
           erisStorageDriveName = "NAS2";
-          ceresIP = "192.168.50.140";
+          ceresIP = "192.168.50.250";
           erisIP = "192.168.50.245";
           deimosIP = "192.168.50.176";
           marsIP = "192.168.50.218";
diff --git a/modules/nixos/default.nix b/modules/nixos/default.nix
index 67f2f85..e6f7b2d 100755
--- a/modules/nixos/default.nix
+++ b/modules/nixos/default.nix
@@ -51,7 +51,7 @@ in
           # comfyui
           # filesorter
           # firefly-iii
-          forgejo
+          # forgejo
           # glance
           # jellyfin
           # logrotate
@@ -64,7 +64,7 @@ in
           # prompter
           # sambaCeres
           # searx
-          # vaultwarden
+          vaultwarden
           # website
           # zookeeper
           # wireguard - moved to systems/ceres/config/wireguard.nix
diff --git a/modules/nixos/services/vaultwarden/default.nix b/modules/nixos/services/vaultwarden/default.nix
index 0a33f11..4eecb4c 100755
--- a/modules/nixos/services/vaultwarden/default.nix
+++ b/modules/nixos/services/vaultwarden/default.nix
@@ -1,5 +1,8 @@
 {
   flake,
+  config,
+  lib,
+  pkgs,
   ...
 }:
 let
@@ -7,169 +10,88 @@ let
   inherit (flake.config.services.instances) vaultwarden smtp web;
   service = vaultwarden;
   host = vaultwarden.domains.url0;
-  secrets = service.secretPaths.path0;
-  localhost = web.localhost.address0;
-  sshPort = 22;
+  vmInterface = service.interface.id;
+  vmMac = service.interface.mac;
+  vmIP = service.interface.ip;
+  vmGateway = service.interface.gate;
+  acmeCertPath = config.security.acme.certs.${host}.directory;
 in
 {
-  microvm = {
-    vms = {
-      vaultwarden = {
-        autostart = true;
-        config =
-          {
-            config,
-            pkgs,
-            lib,
-            ...
-          }:
-          {
-            system.stateVersion = "25.05";
-            time.timeZone = "America/Winnipeg";
+  # Import the microvm host module
+  imports = [
+    # Add microvm.nixosModules.host to your flake inputs if not already present
+  ];
 
-            users.users.root = {
-              openssh.authorizedKeys.keys = flake.config.people.users.${user0}.sshKeys;
-            };
+  # Configure the bridge for microVM networking
+  systemd.network = {
+    enable = true;
 
-            services = {
-              vaultwarden = {
-                enable = true;
-                environmentFile = config.sops.secrets."${service.name}-env".path;
-                config = {
-                  # Domain Configuration
-                  DOMAIN = "https://${host}";
-                  # Email Configuration
-                  SMTP_AUTH_MECHANISM = "Plain";
-                  SMTP_EMBED_IMAGES = true;
-                  SMTP_FROM = smtp.email.address0;
-                  SMTP_FROM_NAME = service.label;
-                  SMTP_HOST = smtp.hostname;
-                  SMTP_PORT = smtp.ports.port1;
-                  SMTP_SECURITY = smtp.records.record1;
-                  SMTP_USERNAME = smtp.email.address0;
-                  # Security Configuration
-                  DISABLE_ADMIN_TOKEN = false;
-                  # Event and Backup Management
-                  EVENTS_DAYS_RETAIN = 90;
-                  # User Features
-                  SENDS_ALLOWED = true;
-                  SIGNUPS_VERIFY = true;
-                  WEB_VAULT_ENABLED = true;
-                  # Rocket (Web Server) Settings
-                  ROCKET_ADDRESS = localhost;
-                  ROCKET_PORT = service.ports.port0;
-                };
-              };
+    # Create bridge interface
+    netdevs."10-virbr0" = {
+      netdevConfig = {
+        Kind = "bridge";
+        Name = "virbr0";
+      };
+    };
 
-              openssh = {
-                enable = true;
-                settings = {
-                  PasswordAuthentication = false;
-                  PermitRootLogin = "prohibit-password";
-                };
-              };
-            };
+    # Configure bridge network
+    networks."10-virbr0" = {
+      matchConfig.Name = "virbr0";
+      addresses = [
+        { Address = "${vmGateway}/24"; }
+      ];
+      networkConfig = {
+        IPv6AcceptRA = false;
+      };
+    };
 
-            systemd = {
-              tmpfiles.rules = [
-                "d ${secrets} 0755 ${service.name} ${service.name} -"
-              ];
-
-              network = {
-                enable = true;
-                networks."10-enp" = {
-                  matchConfig.Name = "enp0s4";
-                  addresses = [ { Address = "${service.interface.ip}/24"; } ];
-                  routes = [
-                    {
-                      Destination = "${localhost}/0";
-                      Gateway = service.interface.gate;
-                    }
-                  ];
-                  dns = [ service.interface.gate ];
-                };
-              };
-            };
-
-            networking.firewall.allowedTCPPorts = [
-              sshPort
-              service.ports.port0
-            ];
-
-            microvm = {
-              vcpu = 2;
-              mem = 2048;
-              hypervisor = "qemu";
-
-              interfaces = [
-                {
-                  type = "tap";
-                  id = service.interface.id;
-                  mac = service.interface.mac;
-                }
-                {
-                  type = "user";
-                  id = service.interface.idUser;
-                  mac = service.interface.macUser;
-                }
-              ];
-
-              shares = [
-                {
-                  mountPoint = "/nix/.ro-store";
-                  proto = "virtiofs";
-                  source = "/nix/store";
-                  tag = "read_only_nix_store";
-                }
-                {
-                  mountPoint = service.varPaths.path0;
-                  proto = "virtiofs";
-                  source = service.mntPaths.path0;
-                  tag = "${service.name}_data";
-                }
-                {
-                  mountPoint = service.secretPaths.path0;
-                  proto = "virtiofs";
-                  source = service.secretPaths.path0;
-                  tag = "${service.name}_secrets";
-                }
-                {
-                  mountPoint = service.ssl.path;
-                  proto = "virtiofs";
-                  source = service.ssl.path;
-                  tag = "acme_certs";
-                }
-              ];
-
-              forwardPorts = [
-                {
-                  from = "host";
-                  host.port = service.interface.ssh;
-                  guest.port = sshPort;
-                }
-              ];
-            };
-          };
+    # Attach tap interfaces to bridge
+    networks."11-microvm" = {
+      matchConfig.Name = "vm-*";
+      networkConfig = {
+        Bridge = "virbr0";
       };
     };
   };
 
-  services.caddy.virtualHosts."${host}" = {
-    extraConfig = ''
-      reverse_proxy ${service.interface.ip}:${toString service.ports.port0} {
-        header_up X-Real-IP {remote_host}
-      }
-      tls ${service.ssl.cert} ${service.ssl.key}
-      encode zstd gzip
-    '';
+  # Enable NAT for microVMs to access internet
+  networking = {
+    nat = {
+      enable = true;
+      internalInterfaces = [ "virbr0" ];
+      externalInterface = lib.mkDefault "enp10s0"; # Use your WireGuard interface by default
+    };
+    firewall = {
+      trustedInterfaces = [ "virbr0" ];
+    };
+    nftables = {
+      enable = true;
+      ruleset = ''
+        table inet filter {
+          chain forward {
+            iifname "virbr0" oifname "virbr0" accept
+          }
+        }
+      '';
+    };
   };
 
+  # Persist microVM data with impermanence
+  environment.persistence."/persist" = {
+    directories = [
+      # Keep existing directories...
+      "/var/lib/microvms"
+      service.varPaths.path0
+    ];
+  };
+
+  # SOPS secrets configuration (only env file needed)
   sops =
     let
       sopsPath = secret: {
-        path = "${secrets}/${service.name}-${secret}";
+        path = "${service.secretPaths.path0}/${service.name}-${secret}";
         owner = "root";
-        mode = "600";
+        mode = "0600";
       };
     in
     {
@@ -184,4 +106,151 @@ in
           ]
       );
     };
+
+  # MicroVM configuration
+  microvm.vms.vaultwarden = {
+    autostart = true;
+    config =
+      {
+        config,
+        pkgs,
+        lib,
+        ...
+      }:
+      {
+        system.stateVersion = "25.05";
+        time.timeZone = "America/Winnipeg";
+
+        # Network configuration
+        systemd.network = {
+          enable = true;
+          networks."20-enp0s5" = {
+            matchConfig.Name = "enp0s5";
+            addresses = [ { Address = "${vmIP}/24"; } ];
+            routes = [
+              {
+                Destination = "0.0.0.0/0";
+                Gateway = vmGateway;
+              }
+            ];
+            dns = [
+              "8.8.8.8"
+              "8.8.4.4"
+            ];
+          };
+        };
+
+        networking = {
+          hostName = "vaultwarden";
+          firewall = {
+            enable = true;
+            allowedTCPPorts = [
+              22
+              service.ports.port0
+            ];
+          };
+        };
+
+        users.users.root = {
+          openssh.authorizedKeys.keys = flake.config.people.users.${user0}.sshKeys;
+        };
+
+        services = {
+          openssh = {
+            enable = true;
+            settings = {
+              PasswordAuthentication = false;
+              PermitRootLogin = "prohibit-password";
+            };
+          };
+
+          vaultwarden = {
+            enable = true;
+            environmentFile = "/run/secrets/${service.name}-env";
+            config = {
+              # Domain Configuration
+              DOMAIN = "https://${host}";
+              # Email Configuration
+              SMTP_AUTH_MECHANISM = "Plain";
+              SMTP_EMBED_IMAGES = true;
+              SMTP_FROM = smtp.email.address0;
+              SMTP_FROM_NAME = service.label;
+              SMTP_HOST = smtp.hostname;
+              SMTP_PORT = smtp.ports.port1;
+              SMTP_SECURITY = smtp.records.record1;
+              SMTP_USERNAME = smtp.email.address0;
+              # Security Configuration
+              DISABLE_ADMIN_TOKEN = false;
+              # Event and Backup Management
+              EVENTS_DAYS_RETAIN = 90;
+              # User Features
+              SENDS_ALLOWED = true;
+              SIGNUPS_VERIFY = true;
+              WEB_VAULT_ENABLED = true;
+              # Rocket (Web Server) Settings
+              ROCKET_ADDRESS = "0.0.0.0";
+              ROCKET_PORT = service.ports.port0;
+            };
+          };
+        };
+
+        # MicroVM-specific configuration
+        microvm = {
+          vcpu = 2;
+          mem = 2048;
+          hypervisor = "qemu";
+
+          interfaces = [
+            {
+              type = "tap";
+              id = vmInterface;
+              mac = vmMac;
+            }
+          ];
+
+          shares = [
+            {
+              mountPoint = "/nix/.ro-store";
+              proto = "virtiofs";
+              source = "/nix/store";
+              tag = "ro-store";
+            }
+            {
+              mountPoint = "/var/lib/vaultwarden";
+              proto = "virtiofs";
+              source = service.varPaths.path0;
+              tag = "vaultwarden-data";
+            }
+            {
+              mountPoint = "/run/secrets";
+              proto = "virtiofs";
+              source = "/run/secrets";
+              tag = "secrets";
+            }
+          ];
+        };
+      };
+  };
+
+  # Caddy reverse proxy configuration on host
+  services.caddy = {
+    enable = true;
+    virtualHosts."${host}" = {
+      extraConfig = ''
+        reverse_proxy ${vmIP}:${toString service.ports.port0} {
+          header_up X-Real-IP {remote_host}
+        }
+        tls ${acmeCertPath}/fullchain.pem ${acmeCertPath}/key.pem
+        encode zstd gzip
+      '';
+    };
+  };
+
+  # Ensure data directory exists
+  systemd.tmpfiles.rules = [
+    "d ${service.varPaths.path0} 0755 root root -"
+  ];
+
+  # Ensure caddy can read ACME certs
+  users.users.caddy.extraGroups = [ "acme" ];
 }
diff --git a/systems/ceres/config/bridge.nix b/systems/ceres/config/bridge.nix
deleted file mode 100755
index cd50a6f..0000000
--- a/systems/ceres/config/bridge.nix
+++ /dev/null
@@ -1,66 +0,0 @@
-{
-  lib,
-  ...
-}:
-{
-  # Enable systemd-networkd for bridge management
-  systemd.network = {
-    enable = true;
-
-    netdevs = {
-      "20-br-vms" = {
-        netdevConfig = {
-          Name = "br-vms";
-          Kind = "bridge";
-        };
-        bridgeConfig = {
-          STP = false;
-        };
-      };
-    };
-
-    networks = {
-      # Connect physical interface to bridge
-      "30-enp10s0" = {
-        matchConfig.Name = "enp10s0";
-        networkConfig = {
-          Bridge = "br-vms";
-          ConfigureWithoutCarrier = true;
-          KeepConfiguration = "yes";
-        };
-        linkConfig = {
-          RequiredForOnline = false;
-        };
-      };
-
-      # Configure bridge to get IP from LAN DHCP
-      "40-br-vms" = {
-        matchConfig.Name = "br-vms";
-        networkConfig = {
-          DHCP = "ipv4";
-          KeepConfiguration = "yes";
-        };
-        linkConfig = {
-          RequiredForOnline = "routable";
-        };
-      };
-
-      # VM tap interface to bridge
-      "50-vm-forgejo" = {
-        matchConfig.Name = "vm-forgejo";
-        networkConfig = {
-          Bridge = "br-vms";
-          ConfigureWithoutCarrier = true;
-        };
-        linkConfig = {
-          RequiredForOnline = false;
-        };
-      };
-    };
-  };
-
-  # IP forwarding (needed for both bridge networking and WireGuard NAT)
-  boot.kernel.sysctl = {
-    "net.ipv4.ip_forward" = 1;
-  };
-}
diff --git a/systems/ceres/config/networking.nix b/systems/ceres/config/networking.nix
index 9b51c1b..5e51bdf 100755
--- a/systems/ceres/config/networking.nix
+++ b/systems/ceres/config/networking.nix
@@ -34,24 +34,6 @@ in
     };
   };
 
-  # Remote rebuild safeguards:
-  # These settings prevent network services from restarting during nixos-rebuild,
-  # which would otherwise drop SSH connections when done remotely.
-  # The bridge configuration changes enp10s0, so we need to prevent systemd-networkd
-  # and NetworkManager from restarting to maintain connectivity.
-
-  # Prevent SSH connections from being killed during network reconfiguration
-  systemd.services.sshd = {
-    stopIfChanged = false;
-    reloadIfChanged = true;
-  };
-
-  # Prevent systemd-networkd from restarting during switches to avoid dropping SSH
-  systemd.services.systemd-networkd = {
-    stopIfChanged = false;
-    restartTriggers = lib.mkForce [ ];
-  };
-
   services = {
     avahi = {
       enable = true;