test: trying to get microVMs to work

This commit is contained in:
Nick 2025-11-10 01:21:38 -06:00
parent 5fc49f405a
commit 72bb3fbe34
11 changed files with 350 additions and 167 deletions

View file

@ -286,7 +286,7 @@ in
varPath = var;
sslPath = "${var}/acme";
sopsPath = "${var}/secrets";
secretPath = "${var}/secrets";
secretPath = "/run/secrets";
cachePath = "/var/cache";
dummy = "";
};

View file

@ -2,24 +2,25 @@
let
inherit (moduleFunctions.instancesFunctions)
domain0
servicePath
sslPath
sopsPath
varPath
mntPath
secretPath
cachePath
;
label = "Firefly-III";
name = "firefly-iii";
domain = "${subdomain}.${domain0}";
subdomain = "finances";
domain = "${subdomain}.${domain0}";
secrets = "${secretPath}/${name}";
ssl = "${sslPath}/${domain}";
in
{
label = label;
name = name;
short = label;
email = {
address0 = "noreply@${domain}";
};
sops = {
path0 = "${sopsPath}/${name}";
address0 = "noreply@${domain0}";
};
domains = {
url0 = domain;
@ -30,14 +31,32 @@ in
"money"
];
subdomain = subdomain;
paths = {
path0 = "${servicePath}/${label}";
};
ports = {
port0 = 8080;
port1 = 8081;
};
interface = {
id = "vm-${name}";
mac = "02:00:00:00:54:04";
idUser = "vmuser-firefly";
macUser = "02:00:00:00:00:04";
ip = "192.168.50.114";
gate = "192.168.50.1";
ssh = 2204;
};
ssl = {
cert = "${sslPath}/${domain}/fullchain.pem";
key = "${sslPath}/${domain}/key.pem";
path = ssl;
cert = "${ssl}/fullchain.pem";
key = "${ssl}/key.pem";
};
varPaths = {
path0 = "${varPath}/${name}";
path1 = "${cachePath}/${name}";
};
mntPaths = {
path0 = "${mntPath}/${name}";
};
secretPaths = {
path0 = secrets;
};
}

View file

@ -37,12 +37,12 @@ in
};
interface = {
id = "vm-${name}";
mac = "02:00:00:00:00:53";
mac = "02:00:00:00:53:03";
idUser = "vmuser-${name}";
macUser = "02:00:00:00:00:05";
ip = "192.168.50.153";
macUser = "02:00:00:00:00:03";
ip = "192.168.50.113";
gate = "192.168.50.1";
ssh = 2205;
ssh = 2203;
};
ssl = {

View file

@ -38,12 +38,12 @@ in
};
interface = {
id = "vm-${name}";
mac = "02:00:00:00:00:52";
mac = "02:00:00:00:52:02";
idUser = "vmuser-${name}";
macUser = "03:00:00:00:00:04";
ip = "192.168.50.152";
macUser = "02:00:00:00:00:02";
ip = "192.168.50.112";
gate = "192.168.50.1";
ssh = 2203;
ssh = 2202;
};
ssl = {
path = ssl;
@ -53,6 +53,7 @@ in
varPaths = {
path0 = "${varPath}/${name}";
path1 = "${cachePath}/${name}";
path2 = "${varPath}/${name}-media";
};
mntPaths = {
path0 = "${mntPath}/${name}";

View file

@ -36,10 +36,10 @@ in
};
interface = {
id = "vm-${name}";
mac = "02:00:00:00:00:51";
mac = "02:00:00:00:51:01";
idUser = "vmuser-vault";
macUser = "02:00:00:00:00:03";
ip = "192.168.50.151";
macUser = "02:00:00:00:00:01";
ip = "192.168.50.111";
gate = "192.168.50.1";
ssh = 2201;
};

View file

@ -49,7 +49,7 @@ in
caddy
# comfyui
# filesorter
# firefly-iii
firefly-iii
forgejo
# glance
jellyfin

View file

@ -1,17 +1,33 @@
{ flake, config, ... }:
{
config,
flake,
...
}:
let
inherit (flake.config.services.instances) firefly-iii smtp;
inherit (flake.config.people) user0;
inherit (flake.config.people.users.${user0}) email;
inherit (flake.config.machines.devices) ceres;
host = service.domains.url0;
service = firefly-iii;
inherit (flake.config.services) instances;
serviceCfg = instances.firefly-iii;
smtpCfg = instances.smtp;
hostCfg = instances.web;
host = serviceCfg.domains.url0;
dns0 = instances.web.dns.provider0;
dns0Path = "dns/${dns0}";
in
{
microvm.vms.${serviceCfg.name} = {
autostart = true;
restartIfChanged = true;
config = {
system.stateVersion = "24.05";
time.timeZone = "America/Winnipeg";
users.users.root.openssh.authorizedKeys.keys = flake.config.people.users.${user0}.sshKeys;
services = {
firefly-iii = {
enable = true;
dataDir = service.paths.path0;
dataDir = serviceCfg.varPaths.path0;
enableNginx = false;
poolConfig = {
"listen.owner" = config.services.caddy.user;
"pm" = "dynamic";
@ -22,52 +38,173 @@ in
"pm.max_requests" = 500;
};
settings = {
DB_CONNECTION = "pgsql";
APP_URL = "https://${host}";
APP_KEY_FILE = config.sops.secrets."${service.name}-pass".path;
DB_PASSWORD_FILE = config.sops.secrets."${service.name}-data".path;
MAIL_MAILER = smtp.name;
MAIL_HOST = smtp.hostname;
MAIL_PORT = smtp.ports.port0;
MAIL_FROM = smtp.email.address0;
MAIL_USERNAME = smtp.email.address0;
MAIL_PASSWORD_FILE = config.sops.secrets."${service.name}-smtp".path;
APP_KEY_FILE = "/run/secrets/pass";
DB_PASSWORD_FILE = "/run/secrets/data";
DB_CONNECTION = "pgsql";
DB_HOST = "db";
DB_DATABASE = "firefly";
DB_USERNAME = "firefly";
MAIL_MAILER = smtpCfg.name;
MAIL_HOST = smtpCfg.hostname;
MAIL_PORT = smtpCfg.ports.port0;
MAIL_FROM = smtpCfg.email.address0;
MAIL_USERNAME = smtpCfg.email.address0;
MAIL_PASSWORD_FILE = "/run/secrets/smtp";
MAIL_ENCRYPTION = "tls";
SITE_OWNER = email.address2;
};
};
firefly-iii-data-importer = {
enable = true;
};
caddy = {
virtualHosts = {
${host} = {
enable = true;
virtualHosts."${serviceCfg.interface.ip}" = {
extraConfig = ''
root * ${config.services.firefly-iii.package}/public
file_server
encode gzip
php_fastcgi unix//run/phpfpm/firefly-iii.sock
tls ${service.ssl.cert} ${service.ssl.key}
'';
file_server
encode gzip
php_fastcgi unix//run/phpfpm/firefly-iii.sock
'';
};
};
};
};
sops =
let
sopsPath = secret: {
path = "${service.sops.path0}/${service.name}-${secret}";
owner = service.name;
mode = "600";
};
in
postgresql = {
enable = true;
ensureDatabases = [ "firefly" ];
ensureUsers = [
{
name = "firefly";
ensureDBOwnership = true;
}
];
};
openssh = {
enable = true;
settings = {
PasswordAuthentication = false;
PermitRootLogin = "prohibit-password";
};
};
};
networking.firewall.allowedTCPPorts = [
22
80
serviceCfg.ports.port0
serviceCfg.ports.port1
];
systemd = {
network = {
enable = true;
networks."20-lan" = {
matchConfig.Name = "enp0s5";
addresses = [
{ Address = "${serviceCfg.interface.ip}/24"; }
];
routes = [
{
Destination = "${hostCfg.localhost.address1}/0";
Gateway = serviceCfg.interface.gate;
}
];
dns = [
"1.1.1.1"
"8.8.8.8"
];
};
};
tmpfiles.rules = [
"Z ${serviceCfg.varPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
"Z ${serviceCfg.secretPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
"Z /var/lib/postgresql 755 postgres postgres -"
];
};
systemd.services.systemd-networkd.wantedBy = [ "multi-user.target" ];
microvm = {
vcpu = 2;
mem = 3072;
hypervisor = "qemu";
interfaces = [
{
type = "tap";
id = serviceCfg.interface.id;
mac = serviceCfg.interface.mac;
}
{
type = "user";
id = serviceCfg.interface.idUser;
mac = serviceCfg.interface.macUser;
}
];
forwardPorts = [
{
from = "host";
host.port = serviceCfg.interface.ssh;
guest.port = 22;
}
];
shares = [
{
mountPoint = "/nix/.ro-store";
proto = "virtiofs";
source = "/nix/store";
tag = "read_only_nix_store";
}
{
mountPoint = "/var/lib/${serviceCfg.name}";
proto = "virtiofs";
source = "${serviceCfg.mntPaths.path0}/data";
tag = "${serviceCfg.name}_data";
}
{
mountPoint = "/var/lib/postgresql";
proto = "virtiofs";
source = "${serviceCfg.mntPaths.path0}/database";
tag = "${serviceCfg.name}_database";
}
{
mountPoint = "/run/secrets";
proto = "virtiofs";
source = "/var/lib/secrets/${serviceCfg.name}";
tag = "host_secrets";
}
];
};
};
};
users.users.caddy.extraGroups = [ "acme" ];
security.acme.certs."${host}" = {
dnsProvider = dns0;
environmentFile = config.sops.secrets.${dns0Path}.path;
group = "caddy";
};
systemd.tmpfiles.rules = [
"d ${serviceCfg.mntPaths.path0} 0751 microvm wheel - -"
];
sops = {
secrets = builtins.listToAttrs (
map
(secret: {
name = "${service.name}-${secret}";
value = sopsPath secret;
name = "${serviceCfg.name}/${secret}";
value = {
path = "/var/lib/secrets";
owner = "root";
mode = "600";
};
})
[
"pass"
@ -76,27 +213,13 @@ in
]
);
};
services.caddy.virtualHosts."${host}" = {
extraConfig = ''
reverse_proxy ${serviceCfg.interface.ip}:80
fileSystems."/var/lib/${service.name}" = {
device = service.paths.path0;
fsType = "none";
options = [
"bind"
];
depends = [
ceres.storage0.mount
];
};
tls ${serviceCfg.ssl.cert} ${serviceCfg.ssl.key}
systemd.tmpfiles.rules = [
"Z ${service.paths.path0} 755 ${service.name} ${service.name} -"
"Z ${service.sops.path0} 755 ${service.name} ${service.name} -"
];
networking = {
firewall.allowedTCPPorts = [
8080
8081
];
encode zstd gzip
'';
};
}

View file

@ -33,14 +33,12 @@ in
forgejo = {
enable = true;
lfs.enable = true;
database = {
type = "postgres";
};
secrets = {
mailer.PASSWD = "/run/secrets/smtp";
};
dump = {
interval = "5:00";
type = "zip";
file = "forgejo-backup";
enable = true;
database.PASSWD = "/run/secrets/database";
};
settings = {
server = {
@ -49,7 +47,7 @@ in
HTTP_PORT = serviceCfg.ports.port0;
};
# If you need to start from scratch, don't forget to turn this off again
service.DISABLE_REGISTRATION = true;
service.DISABLE_REGISTRATION = false;
actions = {
ENABLED = true;
DEFAULT_ACTIONS_URL = "github";
@ -77,6 +75,17 @@ in
PermitRootLogin = "prohibit-password";
};
};
postgresql = {
enable = true;
ensureDatabases = [ serviceCfg.name ];
ensureUsers = [
{
name = serviceCfg.name;
ensureDBOwnership = true;
}
];
};
};
networking.firewall.allowedTCPPorts = [
@ -105,7 +114,7 @@ in
addresses = [ { Address = "${serviceCfg.interface.ip}/24"; } ];
routes = [
{
Destination = "0.0.0.0/0";
Destination = "${hostCfg.localhost.address1}/0";
Gateway = serviceCfg.interface.gate;
}
];
@ -114,10 +123,12 @@ in
"8.8.8.8"
];
};
};
tmpfiles.rules = [
"d ${serviceCfg.varPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
"Z ${serviceCfg.varPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
"Z ${serviceCfg.secretPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
"Z /var/lib/postgresql 0755 postgres postgres -"
];
};
@ -156,13 +167,19 @@ in
{
mountPoint = "/var/lib/${serviceCfg.name}";
proto = "virtiofs";
source = serviceCfg.mntPaths.path0;
source = "${serviceCfg.mntPaths.path0}/data";
tag = "${serviceCfg.name}_data";
}
{
mountPoint = "/var/lib/postgresql";
proto = "virtiofs";
source = "${serviceCfg.mntPaths.path0}/database";
tag = "${serviceCfg.name}_database";
}
{
mountPoint = "/run/secrets";
proto = "virtiofs";
source = "/run/secrets/${serviceCfg.name}";
source = "/var/lib/secrets/${serviceCfg.name}";
tag = "host_secrets";
}
];
@ -171,7 +188,9 @@ in
};
systemd.tmpfiles.rules = [
"d ${serviceCfg.mntPaths.path0} 0777 root root -"
"d ${serviceCfg.mntPaths.path0}/data 0751 microvm wheel - -"
"d ${serviceCfg.mntPaths.path0}/database 0751 microvm wheel - -"
];
services.caddy.virtualHosts."${host}" = {
@ -188,10 +207,12 @@ in
sops.secrets = {
"${serviceCfg.name}/smtp" = {
path = "/var/lib/secrets";
owner = "root";
mode = "0600";
};
"${serviceCfg.name}/database" = {
path = "/var/lib/secrets";
owner = "root";
mode = "0600";
};

View file

@ -17,7 +17,7 @@ in
autostart = true;
restartIfChanged = true;
config = {
system.stateVersion = "24.05";
system.stateVersion = "25.05";
time.timeZone = "America/Winnipeg";
users.users.root.openssh.authorizedKeys.keys = flake.config.people.users.${user0}.sshKeys;
services = {
@ -42,11 +42,20 @@ in
serviceCfg.ports.port2
];
# fileSystems."/tmp" = {
# device = "tmpfs";
# fsType = "tmpfs";
# options = [
# "size=4G"
# "mode=1777"
# ];
# };
systemd = {
network = {
enable = true;
networks."20-lan" = {
matchConfig.Name = "enp0s5";
matchConfig.Name = "enp0s6";
addresses = [ { Address = "${serviceCfg.interface.ip}/24"; } ];
routes = [
{
@ -60,13 +69,16 @@ in
];
};
};
services.systemd-networkd.wantedBy = [ "multi-user.target" ];
tmpfiles.rules = [
"d ${serviceCfg.varPaths.path0}/media 0755 ${serviceCfg.name} ${serviceCfg.name} -"
];
# tmpfiles.rules = [
# "Z ${serviceCfg.varPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
# "Z ${serviceCfg.varPaths.path1} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
# "Z ${serviceCfg.varPaths.path2} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
# ];
};
systemd.services.systemd-networkd.wantedBy = [ "multi-user.target" ];
microvm = {
vcpu = 6;
mem = 8192;
@ -100,7 +112,7 @@ in
{
mountPoint = serviceCfg.varPaths.path0;
proto = "virtiofs";
source = serviceCfg.mntPaths.path0;
source = "${serviceCfg.mntPaths.path0}/data";
tag = "${serviceCfg.name}_data";
}
{
@ -109,15 +121,12 @@ in
source = "${serviceCfg.mntPaths.path0}/cache";
tag = "${serviceCfg.name}_cache";
}
];
};
fileSystems."/tmp" = {
device = "tmpfs";
fsType = "tmpfs";
options = [
"size=4G"
"mode=1777"
{
mountPoint = serviceCfg.varPaths.path2;
proto = "virtiofs";
source = "${serviceCfg.mntPaths.path0}/media";
tag = "${serviceCfg.name}_media";
}
];
};
};
@ -149,8 +158,9 @@ in
users.users.caddy.extraGroups = [ "acme" ];
systemd.tmpfiles.rules = [
"d ${serviceCfg.mntPaths.path0} 0755 999 999 -"
"d ${serviceCfg.mntPaths.path0}/cache 0755 999 999 -"
"d ${serviceCfg.mntPaths.path0} 0751 microvm wheel - -"
"d ${serviceCfg.mntPaths.path0}/data 0751 microvm wheel - -"
"d ${serviceCfg.mntPaths.path0}/cache 0751 microvm wheel - -"
"d ${serviceCfg.mntPaths.path0}/media 0751 microvm wheel - -"
];
}

View file

@ -77,7 +77,8 @@ in
serviceCfg.ports.port0
];
systemd.network = {
systemd = {
network = {
enable = true;
networks."20-lan" = {
matchConfig.Name = "enp0s5";
@ -97,6 +98,13 @@ in
};
};
tmpfiles.rules = [
"Z ${serviceCfg.varPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
"Z ${serviceCfg.secretPaths.path0} 0755 ${serviceCfg.name} ${serviceCfg.name} -"
];
};
systemd.services.systemd-networkd.wantedBy = [ "multi-user.target" ];
microvm = {
@ -140,7 +148,7 @@ in
{
mountPoint = "/run/secrets";
proto = "virtiofs";
source = "/run/secrets/${serviceCfg.name}";
source = "/var/lib/secrets/${serviceCfg.name}";
tag = "host_secrets";
}
];
@ -171,11 +179,12 @@ in
users.users.caddy.extraGroups = [ "acme" ];
systemd.tmpfiles.rules = [
"d ${serviceCfg.mntPaths.path0} 0777 root root -"
"d ${serviceCfg.mntPaths.path0} 0751 microvm wheel - -"
];
sops.secrets = {
"${serviceCfg.name}/env" = {
path = "/var/lib/secrets";
owner = "root";
mode = "0600";
};

View file

@ -45,10 +45,10 @@ caddy:
prompter-auth: ENC[AES256_GCM,data:uEj6gruCfcIRoCQY9eNcOka+PAIIhAlKnI+ehZ88aZo90tINcxZ7ZvKqlTJr4rt5o+EO7rvRJcYH/s8/+piszFyxSa64Rtq5KdAjfHnRm0QM8q/2JIHnZsQC3fPz1S177WPs/c3Eydh4VeVe,iv:ZOru4ABFgIy9DoTlMl3InSf8zM1ERNpbRNLN6vy97Jc=,tag:5v3w7kvFQCEPBjchE8K0cw==,type:str]
comfyui-auth: ENC[AES256_GCM,data:YkHxbW/0zTmnrggXKl2jNO4OnBaepmCwB3ZC6d8MPIKf8snWJzAvTq5+X5ABzziwKaypHRTcS6vuNntxKrrD8DS7hX9DqVCZc5WeFHI6S5VzHh3SprW2MF4E8nm4Hj+VHoKGmRSSOU1cfX3J,iv:v0Pid0BCY2QsMNaahBvJd4WWZD115JDLHlOCQvPiaGU=,tag:gpsAgt052NoOyIa9WqJXyg==,type:str]
wifi-home: ENC[AES256_GCM,data:5NYSCUyalDf7gZF7WaRQJCo=,iv:RkVZKsmVEBg5M28DSkBD41673iLM+dqDAAhSwjqejck=,tag:QQ17VSWOnU0bGglZq6455Q==,type:str]
firefly-iii-pass: ENC[AES256_GCM,data:gy7CuAy2PqKyr/+fHjHuKosj7Mi2cfOop4bLew0vZtH9i6IbTs+3wQb1bk5OMmQgMTbm,iv:R+IK4mEuZMhAz5R+9HyL2zscISc+yrSoipcwMiwNL18=,tag:2GF+O7YKXKgFCUGvbZyQKQ==,type:str]
firefly-iii-data: ENC[AES256_GCM,data:EY/CNiSrnmUjotIshk4KqJ2P7IMpiXYyBr7NeYcI69k=,iv:bocGJHNLMAfHFjs3/6wwxwYqq0qar/uNrwppK+MQjBg=,tag:2H5TD6bd9PUgN7BWkwNuzA==,type:str]
firefly-iii-smtp: ENC[AES256_GCM,data:suCsPpd5acpasLLJPcgf9gUQlz4geqm/fNlw5b1+zMo=,iv:63o2Jtrn1T+CSeB9YZ9Zr0873zxgAdBDklwdNuC2bT4=,tag:L4smPSDq/FHMQzS39ege1Q==,type:str]
roundcube-pass: ENC[AES256_GCM,data:vLvNVgiOQKIIoBhFD2if4Ct/1qugwe6i9OG8rB4sv4o=,iv:iJJlzgIocPe3ty67C39MF09FkU+p7hqd+GLnE0PBJAA=,tag:kzPVQP55YwMeYHrrsHFHJQ==,type:str]
firefly-iii:
pass: ENC[AES256_GCM,data:e7uOZ284bAmokWolrRq81mEguMovZDd9D/7Sr8BwIR6U1Ea/qPfErPgdUEw6OGdQLmmaHg==,iv:2QyCe7cKpys4CEtnNKOBxRA0jo6u2OFU9xeiKnO4myc=,tag:WKCopzEN8+xmAIvla8hqtQ==,type:str]
data: ENC[AES256_GCM,data:921LhcRTWVk24eEAQoDMV+RllSP3PbSXCCIDXlQA80Mq,iv:YXEgas77DgdyPTnBZa/ySjcERBIwmdDZJbijeNKNF24=,tag:Wj25wA7tLJ2bZ/faG9DUhg==,type:str]
smtp: ENC[AES256_GCM,data:+e4MiRZ2WOZyWYpMf+By1Eb45ih4TA+svLI2+00yQk82,iv:+52+kJouMwkOSDEaOCA8V80+wT/VzNxgtCkOO68SCdk=,tag:YrtrJAXIhQpsUTEeYvrVwQ==,type:str]
password-user0: ENC[AES256_GCM,data:VKrySmPAKh3UwCQXJS0EnOPPLDrigWtw5g4WMbSGz/VRtbzlQxMIgs42c/8NnHiqr98ifWy7u9c280oo7SrHhQmEOOvxfITQ9A==,iv:toGkVKCjsmtPP5Ukk/q8kPSmJo3FcTAyj2vcIEkHmU0=,tag:Nhucsk1kgx7zDZZQKycKZQ==,type:str]
sops:
age:
@ -61,7 +61,7 @@ sops:
bXBOa1VSakoyaWxpODJEOU11QUZCaUEK8Ch9Ten3DdrPHF1DTH2qei85AlHUOaLD
aNfzakake7ej+MxJYdKEU0bcWofNMKzIlZa2uM10KZSENDP8d8qlig==
-----END AGE ENCRYPTED FILE-----
lastmodified: "2025-11-09T09:28:13Z"
mac: ENC[AES256_GCM,data:JRE3egK93AnFH+d7MJIZ9a0URE2BV/BJIdfbAWZqdfhicOVYb3zg3iJ2EhLIQ2Nm6UnqZe/t27bveQ222orZ3XmD0K8afS+dakEhGy+TcdBuehUiKQ4WNn8cVmahBWc7cy/WaTY4l9L9YzD5N5v7gJgpPknMpdr6qyyzf+i1vj8=,iv:BVunX3YbWzxqPAScme/5RFka7Lyi6gjZ5miSky61kss=,tag:7ont6bHzZuAxHvctsnUN8A==,type:str]
lastmodified: "2025-11-10T05:21:40Z"
mac: ENC[AES256_GCM,data:1e2wUXNJSCP5108HqTnaEalBrDgGI60CgsLrc/ZWE52OpUxOkzfhXZDb1MZWkPEUbmc3ZeJazF9SHBHspYqnO/wznisEzLAPpGMLJePqSPla6pg81UqZdVawwtqoopurCmtr/SHpdQfGcHk9pR5n7tDdwat7xIEPoOStyZUpoDE=,iv:wg9XTEIr4ozhqr4mQP5auG+DZls1towlDAJSGXb08SI=,tag:PUDxUZfTYqy+FMGd6sLy3g==,type:str]
unencrypted_suffix: _unencrypted
version: 3.11.0