From 6f97e6974500ea46540e36c3686f6f32f4a2ff4a Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 16 Nov 2025 18:40:09 -0600 Subject: [PATCH 1/2] feat: added comfyui --- .../bookmarks/config/flake/selfHosted.nix | 2 - modules/nixos/guests/comfyui/default.nix | 60 +++++++++++++++---- 2 files changed, 47 insertions(+), 15 deletions(-) mode change 100644 => 100755 modules/nixos/guests/comfyui/default.nix diff --git a/modules/home/gui/apps/browsers/floorp/config/bookmarks/config/flake/selfHosted.nix b/modules/home/gui/apps/browsers/floorp/config/bookmarks/config/flake/selfHosted.nix index bf17c0e..0053eb5 100755 --- a/modules/home/gui/apps/browsers/floorp/config/bookmarks/config/flake/selfHosted.nix +++ b/modules/home/gui/apps/browsers/floorp/config/bookmarks/config/flake/selfHosted.nix @@ -36,7 +36,6 @@ let instances.searx.name instances.vaultwarden.name instances.prompter.name - instances.comfyui.name instances.firefly-iii.name ]; } @@ -54,7 +53,6 @@ let name: "http://${ceres.ip.address0}:${builtins.toString instances.${name}.ports.port0}"; services = [ instances.jellyfin.name - instances.comfyui.name instances.firefly-iii.name ]; } diff --git a/modules/nixos/guests/comfyui/default.nix b/modules/nixos/guests/comfyui/default.nix old mode 100644 new mode 100755 index f6993ce..0277c5b --- a/modules/nixos/guests/comfyui/default.nix +++ b/modules/nixos/guests/comfyui/default.nix @@ -140,14 +140,36 @@ in 80 ]; + security.pam.loginLimits = [ + { + domain = "*"; + type = "soft"; + item = "nofile"; + value = "65536"; + } + { + domain = "*"; + type = "hard"; + item = "nofile"; + value = "1048576"; + } + ]; systemd = { - services.docker-comfyui = { - after = [ - "network-online.target" - "docker.service" - ]; - wants = [ "network-online.target" ]; - requires = [ "docker.service" ]; + services = { + docker = { + serviceConfig = { + LimitNOFILE = "infinity"; + LimitNPROC = "infinity"; + }; + }; + docker-comfyui = { + after = [ + "network-online.target" + "docker.service" + ]; + wants = [ "network-online.target" ]; + requires = [ "docker.service" ]; + }; }; network = { enable = true; @@ -179,6 +201,9 @@ in "vm.swappiness" = 1; "vm.dirty_ratio" = 15; "vm.dirty_background_ratio" = 5; + "fs.file-max" = 2097152; + "fs.inotify.max_user_watches" = 524288; + "fs.inotify.max_user_instances" = 512; }; microvm = { @@ -206,6 +231,15 @@ in } ]; + volumes = [ + { + image = "${serviceCfg.mntPaths.path0}/docker-volume.img"; + mountPoint = "/var/lib/docker"; + size = 102400; + autoCreate = true; + } + ]; + shares = [ { mountPoint = "/nix/.ro-store"; @@ -219,12 +253,12 @@ in source = "${serviceCfg.mntPaths.path0}/data"; tag = "${serviceCfg.name}_data"; } - { - mountPoint = "/var/lib/docker"; - proto = "virtiofs"; - source = "${serviceCfg.mntPaths.path0}/docker"; # Store on host - tag = "docker_data"; - } + # { + # mountPoint = "/var/lib/docker"; + # proto = "virtiofs"; + # source = "${serviceCfg.mntPaths.path0}/docker"; # Store on host + # tag = "docker_data"; + # } ]; }; }; From 2fc17f3539be102852320c5842b7290aecc86795 Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 16 Nov 2025 20:25:45 -0600 Subject: [PATCH 2/2] feat: added comfyui --- modules/config/instances/config/comfyui.nix | 25 +- modules/nixos/guests/comfyui/default.nix | 383 +++++++------------- secrets/secrets.yaml | 6 +- 3 files changed, 155 insertions(+), 259 deletions(-) diff --git a/modules/config/instances/config/comfyui.nix b/modules/config/instances/config/comfyui.nix index a14bee8..d5406b5 100755 --- a/modules/config/instances/config/comfyui.nix +++ b/modules/config/instances/config/comfyui.nix @@ -1,11 +1,15 @@ { moduleFunctions }: let inherit (moduleFunctions.instancesFunctions) + domain0 + sslPath varPath mntPath ; label = "ComfyUI"; name = "comfyui"; + domain = "${name}.${domain0}"; + ssl = "${sslPath}/${domain}"; in { label = label; @@ -16,23 +20,24 @@ in "comfy" "ui" ]; + domains = { + url0 = domain; + }; ports = { port0 = 8188; }; - interface = { - id = "vm-comfy"; - mac = "02:00:00:00:54:24"; - idUser = "vmuser-comfy"; - macUser = "02:00:00:00:00:24"; - ip = "192.168.50.214"; - gate = "192.168.50.1"; - ssh = 2304; - }; varPaths = { path0 = "${varPath}/${name}"; }; mntPaths = { path0 = "${mntPath}/${name}"; }; - + secretPaths = { + path0 = "/run/secrets"; + }; + ssl = { + path = ssl; + cert = "${ssl}/fullchain.pem"; + key = "${ssl}/key.pem"; + }; } diff --git a/modules/nixos/guests/comfyui/default.nix b/modules/nixos/guests/comfyui/default.nix index 0277c5b..138e8df 100755 --- a/modules/nixos/guests/comfyui/default.nix +++ b/modules/nixos/guests/comfyui/default.nix @@ -1,273 +1,164 @@ { flake, + config, ... }: let - inherit (flake.config.people) user0; inherit (flake.config.services) instances; serviceCfg = instances.comfyui; - hostCfg = instances.web; + localhost = instances.web.localhost.address1; + host = serviceCfg.domains.url0; + dns = instances.web.dns.provider0; + dnsPath = "dns/${dns}"; + in { - microvm.vms = { - ${serviceCfg.name} = { - autostart = true; - restartIfChanged = true; - config = { - system.stateVersion = "24.05"; - time.timeZone = "America/Winnipeg"; - users.users.root.openssh.authorizedKeys.keys = flake.config.people.users.${user0}.sshKeys; + virtualisation.docker = { + enable = true; + enableNvidia = true; + autoPrune = { + enable = true; + dates = "weekly"; + }; + }; - imports = [ - flake.inputs.filesorter.nixosModules.default - ]; + virtualisation.oci-containers = { + backend = "docker"; + containers.comfyui = { + image = "yanwk/comfyui-boot:cu126-slim"; + autoStart = true; - services = { - filesorter.enable = true; - caddy = { - enable = true; - virtualHosts = { - ":80" = { - extraConfig = '' - # Main reverse proxy with WebSocket support - reverse_proxy ${hostCfg.localhost.address1}:${toString serviceCfg.ports.port0} { - header_up Host {host} - header_up X-Real-IP {remote} - header_up X-Forwarded-For {remote} - header_up X-Forwarded-Proto {scheme} - - # WebSocket support - critical for ComfyUI real-time updates - header_up Connection {>Connection} - header_up Upgrade {>Upgrade} - - # Longer timeouts for generation tasks - transport http { - read_timeout 300s - write_timeout 300s - } - } + ports = [ + "${localhost}:${toString serviceCfg.ports.port0}:8188" + ]; - # Security headers - header { - Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" - X-Frame-Options "DENY" - X-Content-Type-Options "nosniff" - X-XSS-Protection "1; mode=block" - Referrer-Policy "strict-origin-when-cross-origin" - -Server - } + volumes = [ + "${serviceCfg.varPaths.path0}:/root" + "${serviceCfg.varPaths.path0}/models:/root/models" + "${serviceCfg.varPaths.path0}/custom_nodes:/root/custom_nodes" + "${serviceCfg.varPaths.path0}/output:/root/output" + "${serviceCfg.varPaths.path0}/input:/root/input" + "${serviceCfg.varPaths.path0}/user:/root/user" + ]; - # Logging - log { - output file /var/log/caddy/comfyui-access.log - format json - } - ''; - }; - }; - }; - openssh = { - enable = true; - settings = { - PasswordAuthentication = false; - PermitRootLogin = "prohibit-password"; - }; - }; - }; + environment = { + CLI_ARGS = "--listen 0.0.0.0 --port 8188 --preview-method auto --dont-print-server"; + NVIDIA_VISIBLE_DEVICES = "0"; + NVIDIA_DRIVER_CAPABILITIES = "compute,utility,graphics"; + }; - virtualisation = { - docker = { - enable = true; - autoPrune = { - enable = true; - dates = "weekly"; - }; - }; - oci-containers = { - backend = "docker"; - containers.comfyui = { - image = "yanwk/comfyui-boot:cu126-slim"; - autoStart = true; + extraOptions = [ + "--runtime=nvidia" + "--gpus=device=0" + # Memory limits to prevent OOM + "--memory=32g" + "--memory-swap=32g" + "--shm-size=16g" + # Security + "--security-opt=no-new-privileges:true" + # Network + "--network=bridge" + # Health check + "--health-cmd=curl -f http://localhost:8188/ || exit 1" + "--health-interval=30s" + "--health-timeout=10s" + "--health-retries=3" + ]; + }; + }; - ports = [ - "${hostCfg.localhost.address1}:${builtins.toString serviceCfg.ports.port0}:8188" - ]; - - volumes = [ - "${serviceCfg.varPaths.path0}:/root" - "${serviceCfg.varPaths.path0}/models:/root/models" - "${serviceCfg.varPaths.path0}/custom_nodes:/root/custom_nodes" - "${serviceCfg.varPaths.path0}/output:/root/output" - "${serviceCfg.varPaths.path0}/input:/root/input" - "${serviceCfg.varPaths.path0}/user:/root/user" - # Mount the device nodes into the container - "/dev/nvidia0:/dev/nvidia0" - "/dev/nvidiactl:/dev/nvidiactl" - "/dev/nvidia-uvm:/dev/nvidia-uvm" - "/dev/nvidia-modeset:/dev/nvidia-modeset" - ]; - - environment = { - CLI_ARGS = "--listen 0.0.0.0 --port 8188 --preview-method auto --dont-print-server"; - NVIDIA_VISIBLE_DEVICES = "all"; - NVIDIA_DRIVER_CAPABILITIES = "compute,utility,graphics"; - }; - - extraOptions = [ - "--privileged" - "--device=/dev/nvidia0" - "--device=/dev/nvidiactl" - "--device=/dev/nvidia-uvm" - "--device=/dev/nvidia-modeset" - "--memory=32g" - "--memory-swap=32g" - "--shm-size=16g" - "--security-opt=no-new-privileges:true" - "--network=bridge" - "--health-cmd=curl -f http://localhost:8188/ || exit 1" - "--health-interval=30s" - "--health-timeout=10s" - "--health-retries=3" - ]; - }; - }; - }; - - users.users.caddy.extraGroups = [ serviceCfg.name ]; - - networking.firewall.allowedTCPPorts = [ - 22 - 80 - ]; - - security.pam.loginLimits = [ - { - domain = "*"; - type = "soft"; - item = "nofile"; - value = "65536"; + services.caddy = { + enable = true; + virtualHosts = { + "${host}" = { + extraConfig = '' + basic_auth { + {$CADDY_AUTH_USER} {$CADDY_AUTH_PASSWORD_HASH} } - { - domain = "*"; - type = "hard"; - item = "nofile"; - value = "1048576"; + + # Main reverse proxy with WebSocket support + reverse_proxy ${localhost}:${toString serviceCfg.ports.port0} { + header_up Host {host} + header_up X-Real-IP {remote} + header_up X-Forwarded-For {remote} + header_up X-Forwarded-Proto {scheme} + + # WebSocket support - critical for ComfyUI real-time updates + header_up Connection {>Connection} + header_up Upgrade {>Upgrade} + + # Longer timeouts for generation tasks + transport http { + read_timeout 300s + write_timeout 300s + } } - ]; - systemd = { - services = { - docker = { - serviceConfig = { - LimitNOFILE = "infinity"; - LimitNPROC = "infinity"; - }; - }; - docker-comfyui = { - after = [ - "network-online.target" - "docker.service" - ]; - wants = [ "network-online.target" ]; - requires = [ "docker.service" ]; - }; - }; - network = { - enable = true; - networks."20-lan" = { - matchConfig.Name = "enp0s5"; - addresses = [ { Address = "${serviceCfg.interface.ip}/24"; } ]; - routes = [ - { - Destination = "${hostCfg.localhost.address1}/0"; - Gateway = serviceCfg.interface.gate; - } - ]; - dns = [ - "1.1.1.1" - "8.8.8.8" - ]; - }; - }; - tmpfiles.rules = [ - "Z ${serviceCfg.varPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -" - "d /var/lib/docker/tmp/ 755 docker docker -" - "d /var/log/caddy 755 caddy caddy -" - ]; - }; - boot.kernel.sysctl = { - "kernel.shmmax" = 68719476736; - "kernel.shmall" = 4194304; - "vm.swappiness" = 1; - "vm.dirty_ratio" = 15; - "vm.dirty_background_ratio" = 5; - "fs.file-max" = 2097152; - "fs.inotify.max_user_watches" = 524288; - "fs.inotify.max_user_instances" = 512; - }; + tls ${serviceCfg.ssl.cert} ${serviceCfg.ssl.key} - microvm = { - vcpu = 6; - mem = 8192; - hypervisor = "qemu"; - interfaces = [ - { - type = "tap"; - id = serviceCfg.interface.id; - mac = serviceCfg.interface.mac; - } - { - type = "user"; - id = serviceCfg.interface.idUser; - mac = serviceCfg.interface.macUser; - } - ]; + # Security headers + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" + X-Frame-Options "DENY" + X-Content-Type-Options "nosniff" + X-XSS-Protection "1; mode=block" + Referrer-Policy "strict-origin-when-cross-origin" + -Server + } - forwardPorts = [ - { - from = "host"; - host.port = serviceCfg.interface.ssh; - guest.port = 22; - } - ]; - - volumes = [ - { - image = "${serviceCfg.mntPaths.path0}/docker-volume.img"; - mountPoint = "/var/lib/docker"; - size = 102400; - autoCreate = true; - } - ]; - - shares = [ - { - mountPoint = "/nix/.ro-store"; - proto = "virtiofs"; - source = "/nix/store"; - tag = "read_only_nix_store"; - } - { - mountPoint = "/var/lib/${serviceCfg.name}"; - proto = "virtiofs"; - source = "${serviceCfg.mntPaths.path0}/data"; - tag = "${serviceCfg.name}_data"; - } - # { - # mountPoint = "/var/lib/docker"; - # proto = "virtiofs"; - # source = "${serviceCfg.mntPaths.path0}/docker"; # Store on host - # tag = "docker_data"; - # } - ]; - }; + # Logging + log { + output file /var/log/caddy/comfyui-access.log + format json + } + ''; }; }; }; systemd.tmpfiles.rules = [ - "d ${serviceCfg.mntPaths.path0}/data 0751 microvm wheel - -" - "d ${serviceCfg.mntPaths.path0}/docker 0751 microvm wheel - -" - + "d ${serviceCfg.varPaths.path0} 755 root root -" + "d ${serviceCfg.secretPaths.path0}/caddy 755 caddy caddy -" + "d /var/log/caddy 755 caddy caddy -" ]; + + systemd.services.caddy = { + serviceConfig = { + EnvironmentFile = config.sops.secrets."caddy/${serviceCfg.name}-auth".path; + }; + }; + + boot.kernel.sysctl = { + "kernel.shmmax" = 68719476736; + "kernel.shmall" = 4194304; + "vm.swappiness" = 1; + "vm.dirty_ratio" = 15; + "vm.dirty_background_ratio" = 5; + }; + + systemd.services.docker-comfyui = { + after = [ + "network-online.target" + "docker.service" + ]; + wants = [ "network-online.target" ]; + requires = [ "docker.service" ]; + }; + + sops.secrets = { + "caddy/${serviceCfg.name}-auth" = { + owner = "caddy"; + group = "caddy"; + mode = "0400"; + }; + }; + + users.users.caddy.extraGroups = [ "acme" ]; + + security.acme.certs."${host}" = { + dnsProvider = dns; + environmentFile = config.sops.secrets.${dnsPath}.path; + group = "caddy"; + }; + } diff --git a/secrets/secrets.yaml b/secrets/secrets.yaml index 9ff8d1b..186bf68 100755 --- a/secrets/secrets.yaml +++ b/secrets/secrets.yaml @@ -46,7 +46,7 @@ opencloud: projectenv: ENC[AES256_GCM,data:+XCd3xScfxCN1Zl5L+4RAOjpmMPhVLSBtqH2nkEUpXhssy5EU82qAanNmqwiIJ1VrYXYovuu3XOwRKY3Ub1nsR5h1S0KUCwav2zmFKVopxF/5jVNIk6qR8Ggz/fAa1YQSW+SAnrtRGvP0Q1SERlCgnH4isVxNvWPyWCZKIgiX2Enu7hVwsJXKLYDomRWt47zzXNUzw50aFn7xPtXE/AYbMPBa+FweCrCfkaQ6i6jPvkdc6VBYTqIanD0908wB2SJA+1xvY7bYgRVB17/4a/9DuUN5J4xU84TOW7EFkvC/hWhlhC58GqQrOFyAgTP4YJHKGbLVKPlc4fcNMh5+pENpPG2fRDElCaLoJcYe6sYhaCDSegpDR/U9bgzKirnCu/hmdG+NQ3sGK/C89JL2kZT+tVT1u5JWnKGOGvLGQm73QUmnssDZVd8ubNsnd57W7siqAXY3+DN46yLrGgmTfHTRi4x2DKF8VCD9jXOxWsyoLvKYDyz09H9dI72xlCtSmcrFAt7bY7uEAWutrPCf3Kh/gq6oFUAPBEwfqhgnpgGA1vyA6o4zhxl4Rqye5YZMx2uNkxdA4wmk9KB/e7BVR/P04TSXoAV931OX7bnlw3XjSw5NTPEPnpmwZ3VPRGGkz171RiQQp+CkwUr35+DdwFrGazuv3wlwAhM19h9SRn8jikrw6PPGVehYp8mB/FhpNgqV0nM2DfjaBqE3yMfDzXH5b92t4Q=,iv:6mlHq6yh03x/FbZNu+A9QBoV6ALX1rRWuL13ItJWriI=,tag:tK6Ek2fzgPPWT8WCeU1Frw==,type:str] caddy: prompter-auth: ENC[AES256_GCM,data:uEj6gruCfcIRoCQY9eNcOka+PAIIhAlKnI+ehZ88aZo90tINcxZ7ZvKqlTJr4rt5o+EO7rvRJcYH/s8/+piszFyxSa64Rtq5KdAjfHnRm0QM8q/2JIHnZsQC3fPz1S177WPs/c3Eydh4VeVe,iv:ZOru4ABFgIy9DoTlMl3InSf8zM1ERNpbRNLN6vy97Jc=,tag:5v3w7kvFQCEPBjchE8K0cw==,type:str] - comfyui-auth: ENC[AES256_GCM,data:YkHxbW/0zTmnrggXKl2jNO4OnBaepmCwB3ZC6d8MPIKf8snWJzAvTq5+X5ABzziwKaypHRTcS6vuNntxKrrD8DS7hX9DqVCZc5WeFHI6S5VzHh3SprW2MF4E8nm4Hj+VHoKGmRSSOU1cfX3J,iv:v0Pid0BCY2QsMNaahBvJd4WWZD115JDLHlOCQvPiaGU=,tag:gpsAgt052NoOyIa9WqJXyg==,type:str] + comfyui-auth: ENC[AES256_GCM,data:7VTXoRxnD0NyVCFRAjHaZswEUsFuQd/ZIwVfqGPmNNV87hn6CBYWvxvcPPFwe+uw7BmKMt+I66DyKx5ydYENTWxPocyT/rFdgdtWwNoenj+JwsUzegmMbEiH2HCZdiwKj0h1lo142mtA6zkc,iv:xT5XHCj8D4dyvglstE2oqo92fLdscCkaNMux43hJ7nQ=,tag:HgU9wAmjPvfoDXgnorB5yA==,type:str] wifi-home: ENC[AES256_GCM,data:5NYSCUyalDf7gZF7WaRQJCo=,iv:RkVZKsmVEBg5M28DSkBD41673iLM+dqDAAhSwjqejck=,tag:QQ17VSWOnU0bGglZq6455Q==,type:str] firefly-iii: pass: ENC[AES256_GCM,data:WjHcoTuEzEq9pfw4QoqRjI4jhu5VPEMOXlHL0olg9dqUj4EGa1Shv5T/kIxdRFuao0y3zQ==,iv:4/fmFOxxDLzplsNGpSJMQOeoNviZw2c2pFlB1ZkRu+o=,tag:7TQ2q/kEFDU4tZxPx53ebw==,type:str] @@ -64,7 +64,7 @@ sops: bXBOa1VSakoyaWxpODJEOU11QUZCaUEK8Ch9Ten3DdrPHF1DTH2qei85AlHUOaLD aNfzakake7ej+MxJYdKEU0bcWofNMKzIlZa2uM10KZSENDP8d8qlig== -----END AGE ENCRYPTED FILE----- - lastmodified: "2025-11-16T06:36:18Z" - mac: ENC[AES256_GCM,data:KuEUXAKdm15YCrw3+sXxjKPY/sDgmc/FjHI8uc0BpRNb/1OWv0jGbPU9FbOJENRzqTPHLMeh3NEb/YKHFF/iYtdXih+0x+/vbdEi22djot8vM/bI6PdcSO3W94MH9FpGmiMTpy/qTi7Ukj6q8lFJUovApqv8SdftO/hcfzkB9UQ=,iv:7GhoGWwyP3LxxhMc9pqhZhxD7tfavkX0EtxCDnNG10c=,tag:ua77G4zx5shd24gU/RpjUQ==,type:str] + lastmodified: "2025-11-17T02:11:29Z" + mac: ENC[AES256_GCM,data:hDts38zNNXTADzifmtdQhcC8mYE44eqGbnJU8RlXCoK/E7DV9frGqH1UHZFe+47Lk3byw/PF98GgT6u44BNQSQE9+h1D9IIBIqkG2WzeRA5JczW+yFjK/znGzYMmrIbQxqKlGY11KB7kEqi8wlaShhgnaWv2H4/eFGDmiwhv/zg=,iv:02j20UCXTkfx+XgtcF1kma3ljUElofd8tDtNRWC7Wn0=,tag:2JDbn4rqrgQpya1gRAImVg==,type:str] unencrypted_suffix: _unencrypted version: 3.11.0