mirror of
https://gitlab.com/upRootNutrition/dotfiles.git
synced 2025-12-06 21:17:14 -06:00
400 lines
19 KiB
Nix
Executable file
400 lines
19 KiB
Nix
Executable file
# ============================================================================
|
|
# MicroVM Configuration
|
|
# ============================================================================
|
|
# This file contains the complete configuration for running a microVM
|
|
#
|
|
# Architecture Overview:
|
|
# ┌────────────────────────────────────────────────┐
|
|
# │ Host (Ceres - NixOS Server) │
|
|
# │ │
|
|
# │ ┌─────────────┐ ┌─────────────┐ │
|
|
# │ │ Caddy │ │ Bridge │ │
|
|
# │ │ (Reverse │───────▶│ (br-vms) │ │
|
|
# │ │ Proxy) │ │ 192.168.50 │ │
|
|
# │ └─────────────┘ │ .240 │ │
|
|
# │ │ └──────┬──────┘ │
|
|
# │ │ │ │
|
|
# │ │ ┌──────▼──────┐ │
|
|
# │ │ │ TAP │ │
|
|
# │ │ │ (vm-*) │ │
|
|
# │ │ └──────┬──────┘ │
|
|
# │ │ │ │
|
|
# │ ┌─────────▼──────────────────────▼─────────┐ │
|
|
# │ │ │ │
|
|
# │ │ MicroVM │ │
|
|
# │ │ ┌─────────────┐ ┌─────────────┐ │ │
|
|
# │ │ │ Service │ │ enp0s5 │ │ │
|
|
# │ │ │ Service │ │ 192.168.50 │ │ │
|
|
# │ │ │ Port 8085 │ │ .151 │ │ │
|
|
# │ │ └─────────────┘ └─────────────┘ │ │
|
|
# │ │ │ │
|
|
# │ └──────────────────────────────────────────┘ │
|
|
# │ │
|
|
# └────────────────────────────────────────────────┘
|
|
#
|
|
# Network Flow:
|
|
# 1. External request → Router (port forward 443) → Host IP (192.168.50.240)
|
|
# 2. Host Caddy receives HTTPS request on port 443
|
|
# 3. Caddy terminates TLS using ACME certificates
|
|
# 4. Caddy forwards HTTP to VM IP (192.168.50.xxx:xxxx)
|
|
# 5. Request travels through br-vms bridge → TAP interface → VM network
|
|
# 6. Service responds back through the same path
|
|
# ============================================================================
|
|
|
|
{
|
|
config,
|
|
flake,
|
|
...
|
|
}:
|
|
let
|
|
# Pull configuration from centralized instance definitions
|
|
# These are defined in modules/config/instances/config/*.nix
|
|
serviceCfg = flake.config.services.instances.service;
|
|
smtpCfg = flake.config.services.instances.smtp;
|
|
hostCfg = flake.config.services.instances.web;
|
|
inherit (flake.config.people) user0;
|
|
inherit (flake.config.services) instances;
|
|
|
|
# DNS provider configuration for ACME DNS-01 challenge
|
|
dns0 = instances.web.dns.provider0;
|
|
dns0Path = "dns/${dns0}";
|
|
in
|
|
{
|
|
# ============================================================================
|
|
# HOST-SIDE CONFIGURATION
|
|
# ============================================================================
|
|
# The following settings run on the host (Ceres), not inside the VM
|
|
|
|
# Add Caddy user to the ACME group so it can read TLS certificates
|
|
# NixOS ACME module creates certs with group ownership, and Caddy needs
|
|
# read access to serve them over HTTPS
|
|
users.users.caddy.extraGroups = [ "acme" ];
|
|
|
|
# Configure Let's Encrypt SSL certificate for service domain
|
|
# Uses DNS-01 challenge (via dns0 provider) instead of HTTP-01
|
|
# This allows cert generation even when the service isn't publicly accessible yet
|
|
security.acme.certs."${serviceCfg.domains.url0}" = {
|
|
dnsProvider = dns0; # DNS provider (e.g., cloudflare, route53)
|
|
environmentFile = config.sops.secrets.${dns0Path}.path; # API credentials from SOPS
|
|
group = "caddy"; # Allow Caddy to read the certs
|
|
};
|
|
|
|
# ============================================================================
|
|
# MICROVM DEFINITION
|
|
# ============================================================================
|
|
# This section defines the entire VM configuration, including networking,
|
|
# services, and resource allocation
|
|
|
|
microvm.vms.service = {
|
|
# Automatically start VM when host boots
|
|
autostart = true;
|
|
|
|
# Restart VM when configuration changes (during nixos-rebuild)
|
|
restartIfChanged = true;
|
|
|
|
# ============================================================================
|
|
# VM GUEST CONFIGURATION
|
|
# ============================================================================
|
|
# Everything inside this 'config' block runs INSIDE the VM, not on the host
|
|
|
|
config = {
|
|
# NixOS version for the VM system
|
|
system.stateVersion = "24.05";
|
|
time.timeZone = "America/Winnipeg";
|
|
|
|
# Allow SSH access to VM using host user's SSH keys
|
|
# This enables direct SSH via: ssh root@192.168.50.xxx
|
|
users.users.root.openssh.authorizedKeys.keys = flake.config.people.users.${user0}.sshKeys;
|
|
|
|
# ============================================================================
|
|
# service SERVICE (Inside VM)
|
|
# ============================================================================
|
|
# Main application configuration - runs on port xxxx inside the VM
|
|
|
|
services = {
|
|
# ============================================================================
|
|
# VM SSH ACCESS
|
|
# ============================================================================
|
|
# Enable SSH server inside the VM for management and debugging
|
|
|
|
openssh = {
|
|
enable = true;
|
|
settings = {
|
|
PasswordAuthentication = false; # Only allow key-based auth
|
|
PermitRootLogin = "prohibit-password"; # Root login only with keys
|
|
};
|
|
};
|
|
|
|
# ============================================================================
|
|
# VM NETWORK CONFIGURATION (systemd-networkd)
|
|
# ============================================================================
|
|
# This configures the network interface INSIDE the VM
|
|
# The VM sees a network interface called "enp0s5" which connects to the
|
|
# host's TAP interface (vm-service) via the bridge (br-vms)
|
|
};
|
|
|
|
# Open firewall ports inside the VM
|
|
networking.firewall.allowedTCPPorts = [
|
|
serviceCfg.ports.port0 # Service web interface port number
|
|
];
|
|
systemd.network = {
|
|
enable = true; # Enable systemd-networkd for network management
|
|
|
|
networks."20-lan" = {
|
|
# Match the network interface created by QEMU
|
|
# QEMU with q35 machine type typically creates "enp0s5" for the first NIC
|
|
matchConfig.Name = "enp0s5";
|
|
|
|
# Assign static IP address to the VM
|
|
# This IP must be on the same subnet as the host bridge (192.168.50.0/24)
|
|
addresses = [ { Address = "${serviceCfg.interface.ip}/24"; } ]; # 192.168.50.xxx/24
|
|
|
|
# Configure default route to reach the internet
|
|
# All traffic (0.0.0.0/0) goes through the gateway (usually your router)
|
|
routes = [
|
|
{
|
|
Destination = "${hostCfg.localhost.address1}/0";
|
|
Gateway = serviceCfg.interface.gate; # 192.168.50.1
|
|
}
|
|
];
|
|
|
|
# DNS servers for the VM to use
|
|
# Using public DNS (Cloudflare and Google) for reliability
|
|
dns = [
|
|
"1.1.1.1"
|
|
"8.8.8.8"
|
|
];
|
|
};
|
|
};
|
|
|
|
# Explicitly start systemd-networkd service
|
|
# By default, systemd.network.enable creates configs but doesn't start the service
|
|
# This ensures the network is actually configured when the VM boots
|
|
systemd.services.systemd-networkd.wantedBy = [ "multi-user.target" ];
|
|
|
|
# ============================================================================
|
|
# MICROVM HARDWARE CONFIGURATION
|
|
# ============================================================================
|
|
# This section defines the VM's virtual hardware, networking, and storage
|
|
|
|
microvm = {
|
|
# Virtual CPU cores allocated to the VM
|
|
vcpu = 2;
|
|
|
|
# Memory allocated to the VM (in MB)
|
|
mem = 1024;
|
|
|
|
# Hypervisor to use (QEMU with KVM acceleration)
|
|
hypervisor = "qemu";
|
|
|
|
# ============================================================================
|
|
# NETWORK INTERFACES
|
|
# ============================================================================
|
|
# The VM has TWO network interfaces for different purposes:
|
|
|
|
interfaces = [
|
|
# ──────────────────────────────────────────────────────────────────
|
|
# Primary Interface: TAP (for LAN connectivity)
|
|
# ──────────────────────────────────────────────────────────────────
|
|
# TAP creates a virtual ethernet device on the host that acts like
|
|
# a physical network cable connecting the VM to the host's network.
|
|
#
|
|
# Network Path:
|
|
# VM (enp0s5) ← virtio-net → TAP (vm-service) → Bridge (br-vms) → Physical NIC (enp10s0) → LAN
|
|
#
|
|
# The host has a bridge (br-vms) configured in systems/ceres/config/networking.nix
|
|
# that connects:
|
|
# - Physical interface: enp10s0
|
|
# - TAP interfaces: vm-* (this and other VMs)
|
|
#
|
|
# This allows the VM to appear as a separate device on your LAN with
|
|
# its own IP address (192.168.50.xxx)
|
|
{
|
|
type = "tap"; # TAP interface (Layer 2 / Ethernet)
|
|
id = serviceCfg.interface.id; # Interface name on host: "vm-service"
|
|
mac = serviceCfg.interface.mac; # MAC address: "02:00:00:00:00:xx"
|
|
}
|
|
|
|
# ──────────────────────────────────────────────────────────────────
|
|
# Secondary Interface: User-mode networking (for fallback/NAT)
|
|
# ──────────────────────────────────────────────────────────────────
|
|
# User-mode networking (SLIRP) provides outbound internet access via NAT
|
|
# without requiring any host configuration. This is a backup interface.
|
|
#
|
|
# - VM gets a private IP (10.0.2.**) on this interface
|
|
# - Provides internet access even if TAP/bridge isn't working
|
|
# - Used for testing and as a fallback
|
|
# - Cannot receive inbound connections (NAT only)
|
|
{
|
|
type = "user"; # User-mode networking (SLIRP/NAT)
|
|
id = serviceCfg.interface.idUser; # Interface name: "vmuser-*"
|
|
mac = serviceCfg.interface.macUser; # MAC address: "02:00:00:00:00:xx"
|
|
}
|
|
];
|
|
|
|
# ============================================================================
|
|
# PORT FORWARDING (Host → VM)
|
|
# ============================================================================
|
|
# Forward ports from the host to the VM for direct access
|
|
# This allows SSH to the VM via: ssh -p 220x root@localhost (from host)
|
|
#
|
|
# Without this, you'd need to SSH via the VM's LAN IP: ssh root@192.168.50.xxx
|
|
forwardPorts = [
|
|
{
|
|
from = "host"; # Forward from host
|
|
host.port = serviceCfg.interface.ssh; # Host port: 220x
|
|
guest.port = 22; # VM port: 22 (SSH)
|
|
}
|
|
];
|
|
|
|
# ============================================================================
|
|
# SHARED DIRECTORIES (Host → VM)
|
|
# ============================================================================
|
|
# VirtioFS allows sharing directories from host to VM with good performance
|
|
# This is better than network shares (NFS/Samba) for VM-host communication
|
|
#
|
|
# Why use VirtioFS instead of storing everything in the VM?
|
|
# 1. Data persists when VM is recreated (VMs are ephemeral, data isn't)
|
|
# 2. Easy backups (just backup host directories)
|
|
# 3. Data accessible from host for maintenance/migration
|
|
# 4. Share read-only nix store from host (saves space)
|
|
|
|
shares = [
|
|
# ──────────────────────────────────────────────────────────────────
|
|
# Nix Store (Read-Only)
|
|
# ──────────────────────────────────────────────────────────────────
|
|
# Share the host's /nix/store as read-only inside the VM
|
|
# This provides all Nix packages without duplicating data
|
|
# The VM can use all the same packages as the host
|
|
{
|
|
mountPoint = "/nix/.ro-store"; # Mount point in VM
|
|
proto = "virtiofs"; # VirtioFS protocol (fast, modern)
|
|
source = "/nix/store"; # Source on host
|
|
tag = "read_only_nix_store"; # Unique identifier
|
|
}
|
|
|
|
# ──────────────────────────────────────────────────────────────────
|
|
# Service Data (Read-Write)
|
|
# ──────────────────────────────────────────────────────────────────
|
|
# Persistent storage for Service's database and attachments
|
|
# Stored on host at: /mnt/storage/service
|
|
# This data survives VM rebuilds/restarts
|
|
{
|
|
mountPoint = "/var/lib/service"; # Where Service stores its data
|
|
proto = "virtiofs"; # VirtioFS protocol
|
|
source = serviceCfg.mntPaths.path0; # Host: /mnt/storage/service
|
|
tag = "service_data"; # Unique identifier
|
|
}
|
|
|
|
# ──────────────────────────────────────────────────────────────────
|
|
# Secrets (Read-Only)
|
|
# ──────────────────────────────────────────────────────────────────
|
|
# Share secrets managed by SOPS from the host
|
|
# Contains sensitive config like SMTP passwords, admin tokens, etc.
|
|
# SOPS-nix decrypts these on the host, then they're shared to the VM
|
|
{
|
|
mountPoint = "/run/secrets"; # Mount point in VM
|
|
proto = "virtiofs"; # VirtioFS protocol
|
|
source = "/run/secrets"; # Source on host
|
|
tag = "host_secrets"; # Unique identifier
|
|
}
|
|
];
|
|
};
|
|
};
|
|
};
|
|
|
|
# ============================================================================
|
|
# HOST-SIDE STORAGE CONFIGURATION
|
|
# ============================================================================
|
|
# Create necessary directories on the host for VM data
|
|
|
|
systemd.tmpfiles.rules = [
|
|
# Create service data directory on host if it doesn't exist
|
|
# This is where the VM's persistent data is actually stored
|
|
# d = directory, 0755 = permissions, root root = owner/group
|
|
"d ${serviceCfg.mntPaths.path0} 0755 root root -" # /mnt/storage/service
|
|
];
|
|
|
|
# ============================================================================
|
|
# CADDY REVERSE PROXY (Host)
|
|
# ============================================================================
|
|
# Caddy runs on the host and forwards HTTPS traffic to the VM
|
|
#
|
|
# Traffic Flow:
|
|
# Internet → Router:443 → Host:443 (Caddy) → VM:xxxx (Service)
|
|
# ↓
|
|
# TLS Termination
|
|
# (ACME Certs)
|
|
#
|
|
# Why use a reverse proxy instead of exposing VM directly?
|
|
# 1. TLS/SSL termination on the host (easier cert management)
|
|
# 2. Single public IP can serve multiple services
|
|
# 3. Additional security layer (Caddy can add headers, rate limiting, etc.)
|
|
# 4. VM doesn't need to handle TLS complexity
|
|
|
|
services.caddy.virtualHosts."${serviceCfg.domains.url0}" = {
|
|
extraConfig = ''
|
|
reverse_proxy ${serviceCfg.interface.ip}:${toString serviceCfg.ports.port0} {
|
|
header_up X-Real-IP {remote_host}
|
|
}
|
|
|
|
tls ${serviceCfg.ssl.cert} ${serviceCfg.ssl.key}
|
|
|
|
encode zstd gzip
|
|
'';
|
|
};
|
|
|
|
# ============================================================================
|
|
# SECRETS MANAGEMENT (SOPS)
|
|
# ============================================================================
|
|
# Configure secrets that will be decrypted by SOPS-nix on the host
|
|
# then shared to the VM via VirtioFS
|
|
#
|
|
# The service/env file contains sensitive environment variables like:
|
|
# - ADMIN_TOKEN: For accessing the admin panel
|
|
# - DATABASE_URL: Database connection string (if using external DB)
|
|
# - SMTP_PASSWORD: For sending email notifications
|
|
|
|
sops.secrets = {
|
|
"service/env" = {
|
|
owner = "root"; # File owner on host
|
|
mode = "0600"; # Permissions (read/write for owner only)
|
|
};
|
|
};
|
|
}
|
|
|
|
# ============================================================================
|
|
# SUMMARY: How This All Works Together
|
|
# ============================================================================
|
|
#
|
|
# 1. BOOT SEQUENCE:
|
|
# - Host starts and creates br-vms bridge
|
|
# - Host creates TAP interface (vm-service)
|
|
# - Host attaches TAP to bridge
|
|
# - QEMU starts with TAP fds and VirtioFS shares
|
|
# - VM boots and sees enp0s5 network interface
|
|
# - systemd-networkd configures enp0s5 with static IP
|
|
# - Service service starts on port xxxx
|
|
#
|
|
# 2. NETWORK CONNECTIVITY:
|
|
# - VM has IP 192.168.50.xxx on LAN (via TAP/bridge)
|
|
# - Host has IP 192.168.50.240 on LAN (on bridge)
|
|
# - Both can reach each other and the internet
|
|
# - Router forwards port 443 to host IP
|
|
#
|
|
# 3. REQUEST FLOW:
|
|
# External → Router:443 → Host:443 (Caddy) → Bridge → TAP → VM:xxxx (Service)
|
|
# Response follows same path in reverse
|
|
#
|
|
# 4. DATA STORAGE:
|
|
# - VM reads packages from host's /nix/store (shared read-only)
|
|
# - VM writes data to /var/lib/service (actually /mnt/storage/service on host)
|
|
# - VM reads secrets from /run/secrets (shared from host via SOPS)
|
|
#
|
|
# 5. MAINTENANCE:
|
|
# - SSH to VM: ssh root@192.168.50.xxx (from LAN)
|
|
# - SSH to VM: ssh -p 220x root@localhost (from host)
|
|
# - Rebuild: sudo nixos-rebuild switch --flake .#ceres
|
|
# - VM automatically restarts on config changes
|
|
#
|
|
# ============================================================================
|