# ============================================================================ # MicroVM Service Template - Production-Ready Configuration # ============================================================================ # This template is based on proven, working configurations from: # - Vaultwarden (simple service with environment file) # - Forgejo (complex service with separate secret files) # - Jellyfin (media service with multiple mounts and tmpfiles) # # CRITICAL SUCCESS FACTORS (learned from production deployments): # 1. Use serviceCfg.name for all service references (not hardcoded strings) # 2. Secrets MUST use service-specific subdirectories: /run/secrets/${serviceCfg.name}/ # 3. Host directories MUST exist with correct permissions BEFORE VM starts # 4. Use 0777 permissions when VM service runs as non-root user with different UID # 5. systemd.tmpfiles can be used INSIDE the VM for VM-internal directories # 6. Host and VM tmpfiles rules serve different purposes - use both when needed # # Architecture Overview: # ┌────────────────────────────────────────────────┐ # │ Host (NixOS Server) │ # │ │ # │ ┌──────────────┐ ┌──────────────┐ │ # │ │ Caddy │ │ br-vms │ │ # │ │ (Reverse │──────│ Bridge │ │ # │ │ Proxy) │ │ 192.168.50 │ │ # │ │ TLS Term │ │ .240 │ │ # │ └──────────────┘ └──────┬───────┘ │ # │ :443 │ │ # │ │ ┌─────▼──────┐ │ # │ │ │ vm-NAME │ │ # │ │ │ (TAP) │ │ # │ │ └─────┬──────┘ │ # │ │ │ │ # │ ┌─────▼──────────────────────▼────────────┐ │ # │ │ │ │ # │ │ MicroVM Guest │ │ # │ │ ┌────────────┐ ┌────────────┐ │ │ # │ │ │ Service │ │ enp0s5 │ │ │ # │ │ │ :PORT │ │192.168.50 │ │ │ # │ │ │ │ │ .1XX │ │ │ # │ │ └────────────┘ └────────────┘ │ │ # │ │ │ │ # │ │ VirtioFS Mounts: │ │ # │ │ • /nix/.ro-store → Host /nix/store │ │ # │ │ • /var/lib/NAME → Host /mnt/storage │ │ # │ │ • /run/secrets → Host /run/secrets/NAME│ │ # │ └─────────────────────────────────────────┘ │ # │ │ # └────────────────────────────────────────────────┘ # # Network Flow: # 1. Internet → Router:443 (port forward) → Host:443 (Caddy) # 2. Caddy terminates TLS using ACME certificates # 3. Caddy proxies HTTP to VM's LAN IP (e.g., 192.168.50.151:8085) # 4. Request: br-vms → TAP (vm-NAME) → VM enp0s5 → Service # 5. Response follows same path in reverse # # IMPORTANT: Split-DNS for LAN Access # - External users: DNS resolves to public IP → router forwards to host # - Internal users: MUST have DNS resolve to 192.168.50.240 (host bridge IP) # OR use /etc/hosts entries, otherwise NAT hairpinning may fail # # ============================================================================ { config, flake, ... }: let # ============================================================================ # CONFIGURATION REFERENCES # ============================================================================ # These pull from your centralized instance definitions # Located in: modules/config/instances/config/*.nix inherit (flake.config.people) user0; inherit (flake.config.services) instances; # REPLACE 'service' with your actual service name identifier # This should match the attribute name in your instances configuration # Examples: vaultwarden, forgejo, jellyfin, etc. serviceCfg = instances.service; # CHANGE THIS # SMTP configuration (if your service needs email) # Remove this line if your service doesn't use SMTP smtpCfg = instances.smtp; # Host/web configuration for routing and DNS hostCfg = instances.web; # Service domain (e.g., "service.example.com") host = serviceCfg.domains.url0; # DNS provider for ACME DNS-01 challenge dns0 = instances.web.dns.provider0; dns0Path = "dns/${dns0}"; in { # ============================================================================ # HOST-SIDE CONFIGURATION # ============================================================================ # These configurations run on the HOST, not inside the VM # They MUST be configured BEFORE the VM starts # ────────────────────────────────────────────────────────────────────────── # 1. Caddy Group Membership # ────────────────────────────────────────────────────────────────────────── # Allow Caddy to read ACME certificates users.users.caddy.extraGroups = [ "acme" ]; # ────────────────────────────────────────────────────────────────────────── # 2. ACME TLS Certificate # ────────────────────────────────────────────────────────────────────────── # Request Let's Encrypt certificate using DNS-01 challenge security.acme.certs."${host}" = { dnsProvider = dns0; environmentFile = config.sops.secrets.${dns0Path}.path; group = "caddy"; }; # ────────────────────────────────────────────────────────────────────────── # 3. Host Storage Directories # ────────────────────────────────────────────────────────────────────────── # Create directories on the host BEFORE VM starts # The VM will mount these via VirtioFS # # PERMISSION PATTERNS (choose based on your service): # - 0755: Safe default when VM service runs as root # - 0777: Required when VM service runs as non-root with different UID # (e.g., jellyfin runs as UID 999 inside VM) # # EXAMPLES FROM WORKING CONFIGS: # Vaultwarden: "d ${serviceCfg.mntPaths.path0} 0777 root root -" # Forgejo: "d ${serviceCfg.mntPaths.path0} 0777 root root -" # Jellyfin: "d ${serviceCfg.mntPaths.path0} 0777 root root -" # "d ${serviceCfg.mntPaths.path0}/cache 0777 root root -" systemd.tmpfiles.rules = [ # Main data directory "d ${serviceCfg.mntPaths.path0} 0777 root root -" # OPTIONAL: Additional directories if needed (like Jellyfin's cache) # "d ${serviceCfg.mntPaths.path0}/cache 0777 root root -" ]; # ────────────────────────────────────────────────────────────────────────── # 4. Secrets Management (SOPS) # ────────────────────────────────────────────────────────────────────────── # Configure secrets decryption on the host # SOPS-nix will decrypt these to /run/secrets/${serviceCfg.name}/* # # CRITICAL: Always use ${serviceCfg.name} for the path prefix! # This prevents conflicts between multiple VMs # # PATTERN 1: Single environment file (like Vaultwarden) # sops.secrets = { # "${serviceCfg.name}/env" = { # owner = "root"; # mode = "0600"; # }; # }; # # PATTERN 2: Multiple secret files (like Forgejo) # sops.secrets = { # "${serviceCfg.name}/smtp" = { # owner = "root"; # mode = "0600"; # }; # "${serviceCfg.name}/database" = { # owner = "root"; # mode = "0600"; # }; # }; # # PATTERN 3: No secrets (like Jellyfin - if service doesn't need secrets) # sops.secrets = {}; sops.secrets = { # CHOOSE ONE OF THE PATTERNS ABOVE AND UNCOMMENT # "${serviceCfg.name}/env" = { # owner = "root"; # mode = "0600"; # }; }; # ============================================================================ # CADDY REVERSE PROXY (Host-Side) # ============================================================================ # Caddy terminates TLS and proxies to the VM services.caddy.virtualHosts."${host}" = { extraConfig = '' # Forward all requests to the VM's IP and port reverse_proxy ${serviceCfg.interface.ip}:${toString serviceCfg.ports.port0} { # Pass the real client IP to the service header_up X-Real-IP {remote_host} } # Use ACME certificate managed by NixOS tls ${serviceCfg.ssl.cert} ${serviceCfg.ssl.key} # Compress responses encode zstd gzip ''; }; # ============================================================================ # MICROVM DEFINITION # ============================================================================ # Everything below defines the VM's configuration microvm.vms.${serviceCfg.name} = { # VM Lifecycle autostart = true; # Start VM automatically on host boot restartIfChanged = true; # Restart VM when configuration changes # ────────────────────────────────────────────────────────────────────── # VM Guest Configuration # ────────────────────────────────────────────────────────────────────── # Everything inside 'config' runs INSIDE the VM config = { # NixOS version (should match host or be compatible) system.stateVersion = "24.05"; # Timezone (should match host for consistent logging) time.timeZone = "America/Winnipeg"; # SSH Access - allow SSH into VM using host user's keys users.users.root.openssh.authorizedKeys.keys = flake.config.people.users.${user0}.sshKeys; # ──────────────────────────────────────────────────────────────────── # Services Configuration # ──────────────────────────────────────────────────────────────────── services = { # ══════════════════════════════════════════════════════════════════ # YOUR SERVICE CONFIGURATION GOES HERE # ══════════════════════════════════════════════════════════════════ # Choose one of the patterns below based on your service type: # ┌───────────────────────────────────────────────────────────────── # │ PATTERN 1: Simple Service (Vaultwarden-style) # │ - Uses environment file for secrets # │ - Single configuration block # └───────────────────────────────────────────────────────────────── # vaultwarden = { # enable = true; # dbBackend = "sqlite"; # config = { # DOMAIN = "https://${host}"; # # # Email Configuration # SMTP_AUTH_MECHANISM = "Plain"; # SMTP_EMBED_IMAGES = true; # SMTP_FROM = serviceCfg.email.address0; # SMTP_FROM_NAME = serviceCfg.label; # SMTP_HOST = smtpCfg.hostname; # SMTP_PORT = smtpCfg.ports.port1; # SMTP_SECURITY = smtpCfg.records.record1; # SMTP_USERNAME = smtpCfg.email.address0; # # # Security Configuration # DISABLE_ADMIN_TOKEN = false; # # # Web Server Settings # ROCKET_ADDRESS = "0.0.0.0"; # ROCKET_PORT = serviceCfg.ports.port0; # }; # # # Mount secrets from host # environmentFile = "/run/secrets/env"; # }; # ┌───────────────────────────────────────────────────────────────── # │ PATTERN 2: Complex Service (Forgejo-style) # │ - Uses separate secret files # │ - More complex settings structure # └───────────────────────────────────────────────────────────────── # forgejo = { # enable = true; # lfs.enable = true; # # # Separate secret files (not environment variables) # secrets = { # mailer.PASSWD = "/run/secrets/smtp"; # }; # # dump = { # interval = "5:00"; # type = "zip"; # file = "forgejo-backup"; # enable = true; # }; # # settings = { # server = { # DOMAIN = host; # ROOT_URL = "https://${host}/"; # HTTP_PORT = serviceCfg.ports.port0; # }; # # service.DISABLE_REGISTRATION = true; # # actions = { # ENABLED = true; # DEFAULT_ACTIONS_URL = "github"; # }; # # mirror = { # ENABLED = true; # }; # # mailer = { # ENABLED = true; # SMTP_ADDR = smtpCfg.hostname; # FROM = smtpCfg.email.address1; # USER = smtpCfg.email.address1; # PROTOCOL = "${smtpCfg.name}+${smtpCfg.records.record1}"; # SMTP_PORT = smtpCfg.ports.port1; # SEND_AS_PLAIN_TEXT = true; # USE_CLIENT_CERT = false; # }; # }; # }; # ┌───────────────────────────────────────────────────────────────── # │ PATTERN 3: Media Service (Jellyfin-style) # │ - Simple enable, uses openFirewall # │ - No secrets needed # └───────────────────────────────────────────────────────────────── # jellyfin = { # enable = true; # openFirewall = true; # }; # ══════════════════════════════════════════════════════════════════ # SSH Server (for VM management) - ALWAYS INCLUDE # ══════════════════════════════════════════════════════════════════ openssh = { enable = true; settings = { PasswordAuthentication = false; PermitRootLogin = "prohibit-password"; }; }; }; # ──────────────────────────────────────────────────────────────────── # Firewall Configuration # ──────────────────────────────────────────────────────────────────── # Open necessary ports inside the VM # # EXAMPLES: # Vaultwarden: [22, 25, 139, 587, 2525, serviceCfg.ports.port0] # Forgejo: [22, 25, 139, 587, 2525, serviceCfg.ports.port0] # Jellyfin: [22, serviceCfg.ports.port0, port1, port2] networking.firewall.allowedTCPPorts = [ 22 # SSH (always include) serviceCfg.ports.port0 # Main service port # OPTIONAL: SMTP ports if service sends email # 25 # SMTP # 587 # SMTP Submission # 2525 # Alternative SMTP # OPTIONAL: Additional ports (like Jellyfin's discovery ports) # serviceCfg.ports.port1 # serviceCfg.ports.port2 ]; # ──────────────────────────────────────────────────────────────────── # OPTIONAL: Temporary Filesystem # ──────────────────────────────────────────────────────────────────── # Some services need a large /tmp (e.g., Forgejo for large Git operations) # Jellyfin also uses this pattern # # UNCOMMENT IF NEEDED: # fileSystems."/tmp" = { # device = "tmpfs"; # fsType = "tmpfs"; # options = [ # "size=4G" # Adjust size as needed # "mode=1777" # Sticky bit for multi-user access # ]; # }; # ──────────────────────────────────────────────────────────────────── # Systemd Configuration # ──────────────────────────────────────────────────────────────────── systemd = { # Network Configuration (systemd-networkd) # Configure the VM's network interface # # INTERFACE NAME: "enp0s5" is typical for QEMU with q35 machine # To verify: SSH into VM and run: ip link show network = { enable = true; networks."20-lan" = { # Match the network interface created by QEMU matchConfig.Name = "enp0s5"; # Static IP Configuration # MUST be on same subnet as host bridge (e.g., 192.168.50.0/24) addresses = [ { Address = "${serviceCfg.interface.ip}/24"; } ]; # Default route for internet access # PATTERN 1: Use "0.0.0.0/0" (Forgejo) # PATTERN 2: Use "${hostCfg.localhost.address1}/0" (Vaultwarden, Jellyfin) # Both work - choose one routes = [ { Destination = "0.0.0.0/0"; # or "${hostCfg.localhost.address1}/0" Gateway = serviceCfg.interface.gate; } ]; # DNS servers dns = [ "1.1.1.1" # Cloudflare "8.8.8.8" # Google ]; }; }; # OPTIONAL: VM-internal tmpfiles (Jellyfin and Forgejo use this) # This creates directories INSIDE the VM, separate from host tmpfiles # Used when the service needs specific ownership/permissions inside VM # # EXAMPLES: # Jellyfin: "d ${serviceCfg.varPaths.path0}/media 0755 ${serviceCfg.name} ${serviceCfg.name} -" # Forgejo: "d ${serviceCfg.varPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -" # # UNCOMMENT IF NEEDED: # tmpfiles.rules = [ # "d ${serviceCfg.varPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -" # ]; }; # Ensure systemd-networkd starts on boot systemd.services.systemd-networkd.wantedBy = [ "multi-user.target" ]; # ──────────────────────────────────────────────────────────────────── # MicroVM Hardware Configuration # ──────────────────────────────────────────────────────────────────── microvm = { # Virtual CPU cores # Adjust based on service needs: # - Light services (Vaultwarden, Forgejo): 2 cores # - Heavy services (Jellyfin): 6 cores vcpu = 2; # Memory in MB # Adjust based on service needs: # - Light services (Vaultwarden): 3072 MB (3 GB) # - Medium services (Forgejo): 3072 MB (3 GB) # - Heavy services (Jellyfin): 8192 MB (8 GB) mem = 3072; # Hypervisor - QEMU with KVM provides best performance hypervisor = "qemu"; # ────────────────────────────────────────────────────────────────── # Network Interfaces # ────────────────────────────────────────────────────────────────── # All working configs use TWO interfaces: TAP + User-mode interfaces = [ # Primary Interface: TAP (LAN Connectivity) { type = "tap"; id = serviceCfg.interface.id; # e.g., "vm-service" mac = serviceCfg.interface.mac; # e.g., "02:00:00:00:00:51" } # Secondary Interface: User-mode (NAT/Fallback) { type = "user"; id = serviceCfg.interface.idUser; # e.g., "vmuser-service" mac = serviceCfg.interface.macUser; # e.g., "02:00:00:00:01:51" } ]; # ────────────────────────────────────────────────────────────────── # Port Forwarding (Host → VM) # ────────────────────────────────────────────────────────────────── # Forward SSH from host to VM for easy access # Access via: ssh -p 22XX root@localhost (from host) forwardPorts = [ { from = "host"; host.port = serviceCfg.interface.ssh; # e.g., 2201 guest.port = 22; } ]; # ────────────────────────────────────────────────────────────────── # Shared Directories (VirtioFS) # ────────────────────────────────────────────────────────────────── # Share directories from host to VM # # IMPORTANT: All source paths must exist on host BEFORE VM starts # Use systemd.tmpfiles.rules (in host section) to create directories shares = [ # ┌─────────────────────────────────────────────────────────────── # │ Nix Store (Read-Only) - ALWAYS INCLUDE # └─────────────────────────────────────────────────────────────── { mountPoint = "/nix/.ro-store"; proto = "virtiofs"; source = "/nix/store"; tag = "read_only_nix_store"; } # ┌─────────────────────────────────────────────────────────────── # │ Service Data (Read-Write) # └─────────────────────────────────────────────────────────────── # CHOOSE ONE PATTERN: # # PATTERN 1: Direct path (Vaultwarden) # { # mountPoint = "/var/lib/bitwarden_rs"; # proto = "virtiofs"; # source = serviceCfg.mntPaths.path0; # tag = "${serviceCfg.name}_data"; # } # # PATTERN 2: Use serviceCfg.name (Forgejo) # { # mountPoint = "/var/lib/${serviceCfg.name}"; # proto = "virtiofs"; # source = serviceCfg.mntPaths.path0; # tag = "${serviceCfg.name}_data"; # } # # PATTERN 3: Use serviceCfg.varPaths (Jellyfin) # { # mountPoint = serviceCfg.varPaths.path0; # proto = "virtiofs"; # source = serviceCfg.mntPaths.path0; # tag = "${serviceCfg.name}_data"; # } { mountPoint = "/var/lib/${serviceCfg.name}"; # ADJUST THIS proto = "virtiofs"; source = serviceCfg.mntPaths.path0; tag = "${serviceCfg.name}_data"; } # ┌─────────────────────────────────────────────────────────────── # │ OPTIONAL: Additional Data Mounts (like Jellyfin's cache) # └─────────────────────────────────────────────────────────────── # UNCOMMENT IF NEEDED: # { # mountPoint = serviceCfg.varPaths.path1; # proto = "virtiofs"; # source = "${serviceCfg.mntPaths.path0}/cache"; # tag = "${serviceCfg.name}_cache"; # } # ┌─────────────────────────────────────────────────────────────── # │ Secrets (Read-Only) - INCLUDE IF SERVICE NEEDS SECRETS # └─────────────────────────────────────────────────────────────── # CRITICAL: Source must use service-specific subdirectory! # This matches the sops.secrets configuration above # # UNCOMMENT IF SERVICE NEEDS SECRETS: # { # mountPoint = "/run/secrets"; # proto = "virtiofs"; # source = "/run/secrets/${serviceCfg.name}"; # tag = "host_secrets"; # } ]; }; }; }; } # ============================================================================ # QUICK REFERENCE: Pattern Comparison # ============================================================================ # # ┌──────────────┬─────────────┬──────────────┬─────────────┐ # │ Aspect │ Vaultwarden │ Forgejo │ Jellyfin │ # ├──────────────┼─────────────┼──────────────┼─────────────┤ # │ vCPU │ 2 │ 2 │ 6 │ # │ Memory │ 3072 MB │ 3072 MB │ 8192 MB │ # │ Secrets │ env file │ sep. files │ none │ # │ /tmp mount │ no │ yes (4G) │ yes (4G) │ # │ VM tmpfiles │ no │ yes │ yes │ # │ Host perms │ 0777 │ 0777 │ 0777 │ # │ Data mounts │ 1 │ 1 │ 2 │ # │ Secrets mnt │ yes │ yes │ no │ # └──────────────┴─────────────┴──────────────┴─────────────┘ # # ============================================================================ # CHECKLIST: Steps to Create New Service # ============================================================================ # # 1. [ ] Copy this template to modules/nixos/services/YOUR-SERVICE/default.nix # 2. [ ] Replace 'service' with your service name in instances reference # 3. [ ] Uncomment and configure your service in services = { ... } # 4. [ ] Adjust vcpu/mem based on service requirements # 5. [ ] Configure secrets (if needed) - both sops.secrets and shares # 6. [ ] Set correct mountPoint for service data (check service docs) # 7. [ ] Adjust firewall ports based on service needs # 8. [ ] Add /tmp mount if service needs large temporary space # 9. [ ] Test build: sudo nixos-rebuild build --flake .#YOUR-HOST # 10. [ ] Deploy: sudo nixos-rebuild switch --flake .#YOUR-HOST # 11. [ ] Verify TAP exists: ip link show vm-YOUR-SERVICE # 12. [ ] SSH to VM: ssh -p 22XX root@localhost # 13. [ ] Check network in VM: ip addr show enp0s5 # 14. [ ] Test service: curl http://VM-IP:PORT # 15. [ ] Test external access: https://YOUR-SERVICE.example.com # # ============================================================================