dotfiles/modules/nixos/services/template/default.nix

# ============================================================================
# MicroVM Service Template - Production-Ready Configuration
# ============================================================================
# This template is based on proven, working configurations from:
# - Vaultwarden (simple service with environment file)
# - Forgejo (complex service with separate secret files)
# - Jellyfin (media service with multiple mounts and tmpfiles)
#
# CRITICAL SUCCESS FACTORS (learned from production deployments):
# 1. Use serviceCfg.name for all service references (not hardcoded strings)
# 2. Secrets MUST use service-specific subdirectories: /run/secrets/${serviceCfg.name}/
# 3. Host directories MUST exist with correct permissions BEFORE VM starts
# 4. Use 0777 permissions when VM service runs as non-root user with different UID
# 5. systemd.tmpfiles can be used INSIDE the VM for VM-internal directories
# 6. Host and VM tmpfiles rules serve different purposes - use both when needed
#
# Architecture Overview:
# ┌────────────────────────────────────────────────┐
# │ Host (NixOS Server)                            │
# │                                                │
# │  ┌──────────────┐      ┌──────────────┐        │
# │  │    Caddy     │      │   br-vms     │        │
# │  │  (Reverse    │──────│  Bridge      │        │
# │  │   Proxy)     │      │ 192.168.50   │        │
# │  │  TLS Term    │      │    .240      │        │
# │  └──────────────┘      └──────┬───────┘        │
# │       :443                    │                │
# │        │                ┌─────▼──────┐         │
# │        │                │  vm-NAME   │         │
# │        │                │    (TAP)   │         │
# │        │                └─────┬──────┘         │
# │        │                      │                │
# │  ┌─────▼──────────────────────▼────────────┐   │
# │  │                                         │   │
# │  │  MicroVM Guest                          │   │
# │  │  ┌────────────┐      ┌────────────┐     │   │
# │  │  │  Service   │      │  enp0s5    │     │   │
# │  │  │  :PORT     │      │192.168.50  │     │   │
# │  │  │            │      │   .1XX     │     │   │
# │  │  └────────────┘      └────────────┘     │   │
# │  │                                         │   │
# │  │  VirtioFS Mounts:                       │   │
# │  │  • /nix/.ro-store → Host /nix/store     │   │
# │  │  • /var/lib/NAME → Host /mnt/storage    │   │
# │  │  • /run/secrets → Host /run/secrets/NAME│   │
# │  └─────────────────────────────────────────┘   │
# │                                                │
# └────────────────────────────────────────────────┘
#
# Network Flow:
# 1. Internet → Router:443 (port forward) → Host:443 (Caddy)
# 2. Caddy terminates TLS using ACME certificates
# 3. Caddy proxies HTTP to VM's LAN IP (e.g., 192.168.50.151:8085)
# 4. Request: br-vms → TAP (vm-NAME) → VM enp0s5 → Service
# 5. Response follows same path in reverse
#
# IMPORTANT: Split-DNS for LAN Access
# - External users: DNS resolves to public IP → router forwards to host
# - Internal users: MUST have DNS resolve to 192.168.50.240 (host bridge IP)
#   OR use /etc/hosts entries, otherwise NAT hairpinning may fail
#
# ============================================================================

{
  config,
  flake,
  ...
}:
let
  # ============================================================================
  # CONFIGURATION REFERENCES
  # ============================================================================
  # These pull from your centralized instance definitions
  # Located in: modules/config/instances/config/*.nix

  inherit (flake.config.people) user0;
  inherit (flake.config.services) instances;

  # REPLACE 'service' with your actual service name identifier
  # This should match the attribute name in your instances configuration
  # Examples: vaultwarden, forgejo, jellyfin, etc.
  serviceCfg = instances.service; # CHANGE THIS

  # SMTP configuration (if your service needs email)
  # Remove this line if your service doesn't use SMTP
  smtpCfg = instances.smtp;

  # Host/web configuration for routing and DNS
  hostCfg = instances.web;

  # Service domain (e.g., "service.example.com")
  host = serviceCfg.domains.url0;

  # DNS provider for ACME DNS-01 challenge
  dns0 = instances.web.dns.provider0;
  dns0Path = "dns/${dns0}";
in
{
  # ============================================================================
  # HOST-SIDE CONFIGURATION
  # ============================================================================
  # These configurations run on the HOST, not inside the VM
  # They MUST be configured BEFORE the VM starts

  # ──────────────────────────────────────────────────────────────────────────
  # 1. Caddy Group Membership
  # ──────────────────────────────────────────────────────────────────────────
  # Allow Caddy to read ACME certificates
  users.users.caddy.extraGroups = [ "acme" ];

  # ──────────────────────────────────────────────────────────────────────────
  # 2. ACME TLS Certificate
  # ──────────────────────────────────────────────────────────────────────────
  # Request Let's Encrypt certificate using DNS-01 challenge
  security.acme.certs."${host}" = {
    dnsProvider = dns0;
    environmentFile = config.sops.secrets.${dns0Path}.path;
    group = "caddy";
  };

  # ──────────────────────────────────────────────────────────────────────────
  # 3. Host Storage Directories
  # ──────────────────────────────────────────────────────────────────────────
  # Create directories on the host BEFORE VM starts
  # The VM will mount these via VirtioFS
  #
  # PERMISSION PATTERNS (choose based on your service):
  # - 0755: Safe default when VM service runs as root
  # - 0777: Required when VM service runs as non-root with different UID
  #         (e.g., jellyfin runs as UID 999 inside VM)
  #
  # EXAMPLES FROM WORKING CONFIGS:
  # Vaultwarden: "d ${serviceCfg.mntPaths.path0} 0777 root root -"
  # Forgejo:     "d ${serviceCfg.mntPaths.path0} 0777 root root -"
  # Jellyfin:    "d ${serviceCfg.mntPaths.path0} 0777 root root -"
  #              "d ${serviceCfg.mntPaths.path0}/cache 0777 root root -"
  systemd.tmpfiles.rules = [
    # Main data directory
    "d ${serviceCfg.mntPaths.path0} 0777 root root -"

    # OPTIONAL: Additional directories if needed (like Jellyfin's cache)
    # "d ${serviceCfg.mntPaths.path0}/cache 0777 root root -"
  ];

  # ──────────────────────────────────────────────────────────────────────────
  # 4. Secrets Management (SOPS)
  # ──────────────────────────────────────────────────────────────────────────
  # Configure secrets decryption on the host
  # SOPS-nix will decrypt these to /run/secrets/${serviceCfg.name}/*
  #
  # CRITICAL: Always use ${serviceCfg.name} for the path prefix!
  # This prevents conflicts between multiple VMs
  #
  # PATTERN 1: Single environment file (like Vaultwarden)
  # sops.secrets = {
  #   "${serviceCfg.name}/env" = {
  #     owner = "root";
  #     mode = "0600";
  #   };
  # };
  #
  # PATTERN 2: Multiple secret files (like Forgejo)
  # sops.secrets = {
  #   "${serviceCfg.name}/smtp" = {
  #     owner = "root";
  #     mode = "0600";
  #   };
  #   "${serviceCfg.name}/database" = {
  #     owner = "root";
  #     mode = "0600";
  #   };
  # };
  #
  # PATTERN 3: No secrets (like Jellyfin - if service doesn't need secrets)
  # sops.secrets = {};

  sops.secrets = {
    # CHOOSE ONE OF THE PATTERNS ABOVE AND UNCOMMENT
    # "${serviceCfg.name}/env" = {
    #   owner = "root";
    #   mode = "0600";
    # };
  };

  # ============================================================================
  # CADDY REVERSE PROXY (Host-Side)
  # ============================================================================
  # Caddy terminates TLS and proxies to the VM
  services.caddy.virtualHosts."${host}" = {
    extraConfig = ''
      # Forward all requests to the VM's IP and port
      reverse_proxy ${serviceCfg.interface.ip}:${toString serviceCfg.ports.port0} {
        # Pass the real client IP to the service
        header_up X-Real-IP {remote_host}
      }

      # Use ACME certificate managed by NixOS
      tls ${serviceCfg.ssl.cert} ${serviceCfg.ssl.key}

      # Compress responses
      encode zstd gzip
    '';
  };

  # ============================================================================
  # MICROVM DEFINITION
  # ============================================================================
  # Everything below defines the VM's configuration

  microvm.vms.${serviceCfg.name} = {
    # VM Lifecycle
    autostart = true; # Start VM automatically on host boot
    restartIfChanged = true; # Restart VM when configuration changes

    # ──────────────────────────────────────────────────────────────────────
    # VM Guest Configuration
    # ──────────────────────────────────────────────────────────────────────
    # Everything inside 'config' runs INSIDE the VM
    config = {
      # NixOS version (should match host or be compatible)
      system.stateVersion = "24.05";

      # Timezone (should match host for consistent logging)
      time.timeZone = "America/Winnipeg";

      # SSH Access - allow SSH into VM using host user's keys
      users.users.root.openssh.authorizedKeys.keys = flake.config.people.users.${user0}.sshKeys;

      # ────────────────────────────────────────────────────────────────────
      # Services Configuration
      # ────────────────────────────────────────────────────────────────────
      services = {
        # ══════════════════════════════════════════════════════════════════
        # YOUR SERVICE CONFIGURATION GOES HERE
        # ══════════════════════════════════════════════════════════════════
        # Choose one of the patterns below based on your service type:

        # ┌─────────────────────────────────────────────────────────────────
        # │ PATTERN 1: Simple Service (Vaultwarden-style)
        # │ - Uses environment file for secrets
        # │ - Single configuration block
        # └─────────────────────────────────────────────────────────────────
        # vaultwarden = {
        #   enable = true;
        #   dbBackend = "sqlite";
        #   config = {
        #     DOMAIN = "https://${host}";
        #
        #     # Email Configuration
        #     SMTP_AUTH_MECHANISM = "Plain";
        #     SMTP_EMBED_IMAGES = true;
        #     SMTP_FROM = serviceCfg.email.address0;
        #     SMTP_FROM_NAME = serviceCfg.label;
        #     SMTP_HOST = smtpCfg.hostname;
        #     SMTP_PORT = smtpCfg.ports.port1;
        #     SMTP_SECURITY = smtpCfg.records.record1;
        #     SMTP_USERNAME = smtpCfg.email.address0;
        #
        #     # Security Configuration
        #     DISABLE_ADMIN_TOKEN = false;
        #
        #     # Web Server Settings
        #     ROCKET_ADDRESS = "0.0.0.0";
        #     ROCKET_PORT = serviceCfg.ports.port0;
        #   };
        #
        #   # Mount secrets from host
        #   environmentFile = "/run/secrets/env";
        # };

        # ┌─────────────────────────────────────────────────────────────────
        # │ PATTERN 2: Complex Service (Forgejo-style)
        # │ - Uses separate secret files
        # │ - More complex settings structure
        # └─────────────────────────────────────────────────────────────────
        # forgejo = {
        #   enable = true;
        #   lfs.enable = true;
        #
        #   # Separate secret files (not environment variables)
        #   secrets = {
        #     mailer.PASSWD = "/run/secrets/smtp";
        #   };
        #
        #   dump = {
        #     interval = "5:00";
        #     type = "zip";
        #     file = "forgejo-backup";
        #     enable = true;
        #   };
        #
        #   settings = {
        #     server = {
        #       DOMAIN = host;
        #       ROOT_URL = "https://${host}/";
        #       HTTP_PORT = serviceCfg.ports.port0;
        #     };
        #
        #     service.DISABLE_REGISTRATION = true;
        #
        #     actions = {
        #       ENABLED = true;
        #       DEFAULT_ACTIONS_URL = "github";
        #     };
        #
        #     mirror = {
        #       ENABLED = true;
        #     };
        #
        #     mailer = {
        #       ENABLED = true;
        #       SMTP_ADDR = smtpCfg.hostname;
        #       FROM = smtpCfg.email.address1;
        #       USER = smtpCfg.email.address1;
        #       PROTOCOL = "${smtpCfg.name}+${smtpCfg.records.record1}";
        #       SMTP_PORT = smtpCfg.ports.port1;
        #       SEND_AS_PLAIN_TEXT = true;
        #       USE_CLIENT_CERT = false;
        #     };
        #   };
        # };

        # ┌─────────────────────────────────────────────────────────────────
        # │ PATTERN 3: Media Service (Jellyfin-style)
        # │ - Simple enable, uses openFirewall
        # │ - No secrets needed
        # └─────────────────────────────────────────────────────────────────
        # jellyfin = {
        #   enable = true;
        #   openFirewall = true;
        # };

        # ══════════════════════════════════════════════════════════════════
        # SSH Server (for VM management) - ALWAYS INCLUDE
        # ══════════════════════════════════════════════════════════════════
        openssh = {
          enable = true;
          settings = {
            PasswordAuthentication = false;
            PermitRootLogin = "prohibit-password";
          };
        };
      };

      # ────────────────────────────────────────────────────────────────────
      # Firewall Configuration
      # ────────────────────────────────────────────────────────────────────
      # Open necessary ports inside the VM
      #
      # EXAMPLES:
      # Vaultwarden: [22, 25, 139, 587, 2525, serviceCfg.ports.port0]
      # Forgejo:     [22, 25, 139, 587, 2525, serviceCfg.ports.port0]
      # Jellyfin:    [22, serviceCfg.ports.port0, port1, port2]
      networking.firewall.allowedTCPPorts = [
        22 # SSH (always include)
        serviceCfg.ports.port0 # Main service port

        # OPTIONAL: SMTP ports if service sends email
        # 25    # SMTP
        # 587   # SMTP Submission
        # 2525  # Alternative SMTP

        # OPTIONAL: Additional ports (like Jellyfin's discovery ports)
        # serviceCfg.ports.port1
        # serviceCfg.ports.port2
      ];

      # ────────────────────────────────────────────────────────────────────
      # OPTIONAL: Temporary Filesystem
      # ────────────────────────────────────────────────────────────────────
      # Some services need a large /tmp (e.g., Forgejo for large Git operations)
      # Jellyfin also uses this pattern
      #
      # UNCOMMENT IF NEEDED:
      # fileSystems."/tmp" = {
      #   device = "tmpfs";
      #   fsType = "tmpfs";
      #   options = [
      #     "size=4G"      # Adjust size as needed
      #     "mode=1777"    # Sticky bit for multi-user access
      #   ];
      # };

      # ────────────────────────────────────────────────────────────────────
      # Systemd Configuration
      # ────────────────────────────────────────────────────────────────────
      systemd = {
        # Network Configuration (systemd-networkd)
        # Configure the VM's network interface
        #
        # INTERFACE NAME: "enp0s5" is typical for QEMU with q35 machine
        # To verify: SSH into VM and run: ip link show
        network = {
          enable = true;

          networks."20-lan" = {
            # Match the network interface created by QEMU
            matchConfig.Name = "enp0s5";

            # Static IP Configuration
            # MUST be on same subnet as host bridge (e.g., 192.168.50.0/24)
            addresses = [
              { Address = "${serviceCfg.interface.ip}/24"; }
            ];

            # Default route for internet access
            # PATTERN 1: Use "0.0.0.0/0" (Forgejo)
            # PATTERN 2: Use "${hostCfg.localhost.address1}/0" (Vaultwarden, Jellyfin)
            # Both work - choose one
            routes = [
              {
                Destination = "0.0.0.0/0"; # or "${hostCfg.localhost.address1}/0"
                Gateway = serviceCfg.interface.gate;
              }
            ];

            # DNS servers
            dns = [
              "1.1.1.1" # Cloudflare
              "8.8.8.8" # Google
            ];
          };
        };

        # OPTIONAL: VM-internal tmpfiles (Jellyfin and Forgejo use this)
        # This creates directories INSIDE the VM, separate from host tmpfiles
        # Used when the service needs specific ownership/permissions inside VM
        #
        # EXAMPLES:
        # Jellyfin: "d ${serviceCfg.varPaths.path0}/media 0755 ${serviceCfg.name} ${serviceCfg.name} -"
        # Forgejo:  "d ${serviceCfg.varPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
        #
        # UNCOMMENT IF NEEDED:
        # tmpfiles.rules = [
        #   "d ${serviceCfg.varPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
        # ];
      };

      # Ensure systemd-networkd starts on boot
      systemd.services.systemd-networkd.wantedBy = [ "multi-user.target" ];

      # ────────────────────────────────────────────────────────────────────
      # MicroVM Hardware Configuration
      # ────────────────────────────────────────────────────────────────────
      microvm = {
        # Virtual CPU cores
        # Adjust based on service needs:
        # - Light services (Vaultwarden, Forgejo): 2 cores
        # - Heavy services (Jellyfin): 6 cores
        vcpu = 2;

        # Memory in MB
        # Adjust based on service needs:
        # - Light services (Vaultwarden): 3072 MB (3 GB)
        # - Medium services (Forgejo): 3072 MB (3 GB)
        # - Heavy services (Jellyfin): 8192 MB (8 GB)
        mem = 3072;

        # Hypervisor - QEMU with KVM provides best performance
        hypervisor = "qemu";

        # ──────────────────────────────────────────────────────────────────
        # Network Interfaces
        # ──────────────────────────────────────────────────────────────────
        # All working configs use TWO interfaces: TAP + User-mode
        interfaces = [
          # Primary Interface: TAP (LAN Connectivity)
          {
            type = "tap";
            id = serviceCfg.interface.id; # e.g., "vm-service"
            mac = serviceCfg.interface.mac; # e.g., "02:00:00:00:00:51"
          }

          # Secondary Interface: User-mode (NAT/Fallback)
          {
            type = "user";
            id = serviceCfg.interface.idUser; # e.g., "vmuser-service"
            mac = serviceCfg.interface.macUser; # e.g., "02:00:00:00:01:51"
          }
        ];

        # ──────────────────────────────────────────────────────────────────
        # Port Forwarding (Host → VM)
        # ──────────────────────────────────────────────────────────────────
        # Forward SSH from host to VM for easy access
        # Access via: ssh -p 22XX root@localhost (from host)
        forwardPorts = [
          {
            from = "host";
            host.port = serviceCfg.interface.ssh; # e.g., 2201
            guest.port = 22;
          }
        ];

        # ──────────────────────────────────────────────────────────────────
        # Shared Directories (VirtioFS)
        # ──────────────────────────────────────────────────────────────────
        # Share directories from host to VM
        #
        # IMPORTANT: All source paths must exist on host BEFORE VM starts
        # Use systemd.tmpfiles.rules (in host section) to create directories
        shares = [
          # ┌───────────────────────────────────────────────────────────────
          # │ Nix Store (Read-Only) - ALWAYS INCLUDE
          # └───────────────────────────────────────────────────────────────
          {
            mountPoint = "/nix/.ro-store";
            proto = "virtiofs";
            source = "/nix/store";
            tag = "read_only_nix_store";
          }

          # ┌───────────────────────────────────────────────────────────────
          # │ Service Data (Read-Write)
          # └───────────────────────────────────────────────────────────────
          # CHOOSE ONE PATTERN:
          #
          # PATTERN 1: Direct path (Vaultwarden)
          # {
          #   mountPoint = "/var/lib/bitwarden_rs";
          #   proto = "virtiofs";
          #   source = serviceCfg.mntPaths.path0;
          #   tag = "${serviceCfg.name}_data";
          # }
          #
          # PATTERN 2: Use serviceCfg.name (Forgejo)
          # {
          #   mountPoint = "/var/lib/${serviceCfg.name}";
          #   proto = "virtiofs";
          #   source = serviceCfg.mntPaths.path0;
          #   tag = "${serviceCfg.name}_data";
          # }
          #
          # PATTERN 3: Use serviceCfg.varPaths (Jellyfin)
          # {
          #   mountPoint = serviceCfg.varPaths.path0;
          #   proto = "virtiofs";
          #   source = serviceCfg.mntPaths.path0;
          #   tag = "${serviceCfg.name}_data";
          # }

          {
            mountPoint = "/var/lib/${serviceCfg.name}"; # ADJUST THIS
            proto = "virtiofs";
            source = serviceCfg.mntPaths.path0;
            tag = "${serviceCfg.name}_data";
          }

          # ┌───────────────────────────────────────────────────────────────
          # │ OPTIONAL: Additional Data Mounts (like Jellyfin's cache)
          # └───────────────────────────────────────────────────────────────
          # UNCOMMENT IF NEEDED:
          # {
          #   mountPoint = serviceCfg.varPaths.path1;
          #   proto = "virtiofs";
          #   source = "${serviceCfg.mntPaths.path0}/cache";
          #   tag = "${serviceCfg.name}_cache";
          # }

          # ┌───────────────────────────────────────────────────────────────
          # │ Secrets (Read-Only) - INCLUDE IF SERVICE NEEDS SECRETS
          # └───────────────────────────────────────────────────────────────
          # CRITICAL: Source must use service-specific subdirectory!
          # This matches the sops.secrets configuration above
          #
          # UNCOMMENT IF SERVICE NEEDS SECRETS:
          # {
          #   mountPoint = "/run/secrets";
          #   proto = "virtiofs";
          #   source = "/run/secrets/${serviceCfg.name}";
          #   tag = "host_secrets";
          # }
        ];
      };
    };
  };
}

# ============================================================================
# QUICK REFERENCE: Pattern Comparison
# ============================================================================
#
# ┌──────────────┬─────────────┬──────────────┬─────────────┐
# │ Aspect       │ Vaultwarden │ Forgejo      │ Jellyfin    │
# ├──────────────┼─────────────┼──────────────┼─────────────┤
# │ vCPU         │ 2           │ 2            │ 6           │
# │ Memory       │ 3072 MB     │ 3072 MB      │ 8192 MB     │
# │ Secrets      │ env file    │ sep. files   │ none        │
# │ /tmp mount   │ no          │ yes (4G)     │ yes (4G)    │
# │ VM tmpfiles  │ no          │ yes          │ yes         │
# │ Host perms   │ 0777        │ 0777         │ 0777        │
# │ Data mounts  │ 1           │ 1            │ 2           │
# │ Secrets mnt  │ yes         │ yes          │ no          │
# └──────────────┴─────────────┴──────────────┴─────────────┘
#
# ============================================================================
# CHECKLIST: Steps to Create New Service
# ============================================================================
#
# 1. [ ] Copy this template to modules/nixos/services/YOUR-SERVICE/default.nix
# 2. [ ] Replace 'service' with your service name in instances reference
# 3. [ ] Uncomment and configure your service in services = { ... }
# 4. [ ] Adjust vcpu/mem based on service requirements
# 5. [ ] Configure secrets (if needed) - both sops.secrets and shares
# 6. [ ] Set correct mountPoint for service data (check service docs)
# 7. [ ] Adjust firewall ports based on service needs
# 8. [ ] Add /tmp mount if service needs large temporary space
# 9. [ ] Test build: sudo nixos-rebuild build --flake .#YOUR-HOST
# 10. [ ] Deploy: sudo nixos-rebuild switch --flake .#YOUR-HOST
# 11. [ ] Verify TAP exists: ip link show vm-YOUR-SERVICE
# 12. [ ] SSH to VM: ssh -p 22XX root@localhost
# 13. [ ] Check network in VM: ip addr show enp0s5
# 14. [ ] Test service: curl http://VM-IP:PORT
# 15. [ ] Test external access: https://YOUR-SERVICE.example.com
#
# ============================================================================