upRootNutrition/dotfiles
upRootNutrition/dotfiles
1
0
Fork 0
mirror of https://gitlab.com/upRootNutrition/dotfiles.git synced 2025-12-07 21:42:16 -06:00
Code Issues Projects 1 Releases Packages Wiki Activity Actions

test: vaultwarden microVM

Browse source
This commit is contained in:
Nick 2025-11-07 13:36:30 -06:00
parent e90d05f83d
commit 7ba592c0c5
43 changed files with 4005 additions and 267 deletions
Download patch file Download diff file Expand all files Collapse all files

128
example/microvm/asserts.nix Executable file
View file

@ -0,0 +1,128 @@
{ config, lib, ... }:
let
inherit (config.networking) hostName;
in
lib.mkIf config.microvm.guest.enable {
assertions =
# check for duplicate volume images
map (volumes: {
assertion = builtins.length volumes == 1;
message = ''
MicroVM ${hostName}: volume image "${(builtins.head volumes).image}" is used ${toString (builtins.length volumes)} > 1 times.
'';
}) (builtins.attrValues (builtins.groupBy ({ image, ... }: image) config.microvm.volumes))
++
# check for duplicate interface ids
map (interfaces: {
assertion = builtins.length interfaces == 1;
message = ''
MicroVM ${hostName}: interface id "${(builtins.head interfaces).id}" is used ${toString (builtins.length interfaces)} > 1 times.
'';
}) (builtins.attrValues (builtins.groupBy ({ id, ... }: id) config.microvm.interfaces))
++
# check for bridge interfaces
map (
{
id,
type,
bridge,
...
}:
if type == "bridge" then
{
assertion = bridge != null;
message = ''
MicroVM ${hostName}: interface ${id} is of type "bridge"
but doesn't have a bridge to attach to defined.
'';
}
else
{
assertion = bridge == null;
message = ''
MicroVM ${hostName}: interface ${id} is not of type "bridge"
and therefore shouldn't have a "bridge" option defined.
'';
}
) config.microvm.interfaces
++
# check for interface name length
map (
{ id, ... }:
{
assertion = builtins.stringLength id <= 15;
message = ''
MicroVM ${hostName}: interface name ${id} is longer than the
the maximum length of 15 characters on Linux.
'';
}
) config.microvm.interfaces
++
# check for duplicate share tags
map (shares: {
assertion = builtins.length shares == 1;
message = ''
MicroVM ${hostName}: share tag "${(builtins.head shares).tag}" is used ${toString (builtins.length shares)} > 1 times.
'';
}) (builtins.attrValues (builtins.groupBy ({ tag, ... }: tag) config.microvm.shares))
++
# check for duplicate share sockets
map
(shares: {
assertion = builtins.length shares == 1;
message = ''
MicroVM ${hostName}: share socket "${(builtins.head shares).socket}" is used ${toString (builtins.length shares)} > 1 times.
'';
})
(
builtins.attrValues (
builtins.groupBy ({ socket, ... }: toString socket) (
builtins.filter ({ proto, ... }: proto == "virtiofs") config.microvm.shares
)
)
)
++
# check for virtiofs shares without socket
map (
{ tag, socket, ... }:
{
assertion = socket != null;
message = ''
MicroVM ${hostName}: virtiofs share with tag "${tag}" is missing a `socket` path.
'';
}
) (builtins.filter ({ proto, ... }: proto == "virtiofs") config.microvm.shares)
++
# blacklist forwardPorts
[
{
assertion =
config.microvm.forwardPorts != [ ]
-> (
config.microvm.hypervisor == "qemu"
&& builtins.any ({ type, ... }: type == "user") config.microvm.interfaces
);
message = ''
MicroVM ${hostName}: `config.microvm.forwardPorts` works only with qemu and one network interface with `type = "user"`
'';
}
]
++
# cloud-hypervisor specific asserts
lib.optionals (config.microvm.hypervisor == "cloud-hypervisor") [
{
assertion =
!(lib.any (str: lib.hasInfix "oem_strings" str) config.microvm.cloud-hypervisor.platformOEMStrings);
message = ''
MicroVM ${hostName}: `config.microvm.cloud-hypervisor.platformOEMStrings` items must not contain `oem_strings`
'';
}
];
warnings =
# 32 MB is just an optimistic guess, not based on experience
lib.optional (config.microvm.mem < 32) ''
MicroVM ${hostName}: ${toString config.microvm.mem} MB of RAM is uncomfortably narrow.
'';
}

69
example/microvm/boot-disk.nix Executable file
View file

@ -0,0 +1,69 @@
{
config,
lib,
pkgs,
...
}:
let
inherit (config.system.boot.loader) kernelFile;
inherit (config.microvm) initrdPath;
kernelPath = "${config.microvm.kernel}/${kernelFile}";
in
{
options.microvm = with lib; {
bootDisk = mkOption {
type = types.path;
description = ''
Generated.
Required for Hypervisors that do not support direct
kernel+initrd loading.
'';
};
};
config = lib.mkIf config.microvm.guest.enable {
microvm.bootDisk =
pkgs.runCommandLocal "microvm-bootdisk.img"
{
nativeBuildInputs = with pkgs; [
parted
libguestfs
];
LIBGUESTFS_PATH = pkgs.libguestfs-appliance;
}
''
# kernel + initrd + slack, in sectors
EFI_SIZE=$(( ( ( $(stat -c %s ${kernelPath}) + $(stat -c %s ${initrdPath}) + 16 * 4096 ) / ( 2048 * 512 ) + 1 ) * 2048 ))
truncate -s $(( ( $EFI_SIZE + 2048 + 33 ) * 512 )) $out
echo Creating partition table
parted --script $out -- \
mklabel gpt \
mkpart ESP fat32 2048s $(( $EFI_SIZE + 2048 - 1 ))"s" \
set 1 boot on
echo Creating EFI partition
export HOME=`pwd`
guestfish --add $out run \: mkfs fat /dev/sda1
guestfs() {
guestfish --add $out --mount /dev/sda1:/ $@
}
guestfs mkdir /loader
echo 'default *.conf' > loader.conf
guestfs copy-in loader.conf /loader/
guestfs mkdir /loader/entries
cat > entry.conf <<EOF
title microvm.nix (${config.system.nixos.label})
linux /${kernelFile}
initrd /${baseNameOf initrdPath}
EOF
guestfs copy-in entry.conf /loader/entries/
guestfs copy-in ${kernelPath} /
guestfs copy-in ${initrdPath} /
'';
};
}

44
example/microvm/default.nix Executable file
View file

@ -0,0 +1,44 @@
{
config,
lib,
pkgs,
...
}:
let
microvm-lib = import ../../lib {
inherit lib;
};
in
{
imports = [
./boot-disk.nix
./store-disk.nix
./options.nix
./asserts.nix
./system.nix
./mounts.nix
./interfaces.nix
./pci-devices.nix
./virtiofsd
./graphics.nix
./optimization.nix
./ssh-deploy.nix
];
config = {
microvm.runner = lib.genAttrs microvm-lib.hypervisors (
hypervisor:
microvm-lib.buildRunner {
inherit pkgs;
microvmConfig = config.microvm // {
inherit (config.networking) hostName;
inherit hypervisor;
};
inherit (config.system.build) toplevel;
}
);
};
}

37
example/microvm/graphics.nix Executable file
View file

@ -0,0 +1,37 @@
{
config,
lib,
pkgs,
...
}:
let
# TODO: did not get sommelier to work
# run-sommelier = with pkgs; writeShellScriptBin "run-sommelier" ''
# exec ${lib.getExe sommelier} --virtgpu-channel -- $@
# '';
# Working: run Wayland applications prefixed with `run-wayland-proxy`
run-wayland-proxy =
with pkgs;
writeShellScriptBin "run-wayland-proxy" ''
exec ${lib.getExe wayland-proxy-virtwl} --virtio-gpu -- $@
'';
# Waypipe. Needs `microvm#waypipe-client` on the host.
run-waypipe =
with pkgs;
writeShellScriptBin "run-waypipe" ''
exec ${lib.getExe waypipe}/bin/waypipe --vsock -s 2:6000 server $@
'';
in
lib.mkIf config.microvm.graphics.enable {
boot.kernelModules = [
"drm"
"virtio_gpu"
];
environment.systemPackages = [
#run-sommelier
run-wayland-proxy
run-waypipe
];
}

87
example/microvm/interfaces.nix Executable file
View file

@ -0,0 +1,87 @@
{
config,
lib,
pkgs,
...
}:
let
interfacesByType =
wantedType: builtins.filter ({ type, ... }: type == wantedType) config.microvm.interfaces;
tapInterfaces = interfacesByType "tap";
macvtapInterfaces = interfacesByType "macvtap";
tapFlags = lib.concatStringsSep " " (
[ "vnet_hdr" ] ++ lib.optional config.microvm.declaredRunner.passthru.tapMultiQueue "multi_queue"
);
# TODO: don't hardcode but obtain from host config
user = "microvm";
group = "kvm";
in
{
microvm.binScripts = lib.mkMerge [
(lib.mkIf (tapInterfaces != [ ]) {
tap-up = ''
set -eou pipefail
''
+ lib.concatMapStrings (
{ id, ... }:
''
if [ -e /sys/class/net/${id} ]; then
${lib.getExe' pkgs.iproute2 "ip"} link delete '${id}'
fi
${lib.getExe' pkgs.iproute2 "ip"} tuntap add name '${id}' mode tap user '${user}' ${tapFlags}
${lib.getExe' pkgs.iproute2 "ip"} link set '${id}' up
''
) tapInterfaces;
tap-down = ''
set -ou pipefail
''
+ lib.concatMapStrings (
{ id, ... }:
''
${lib.getExe' pkgs.iproute2 "ip"} link delete '${id}'
''
) tapInterfaces;
})
(lib.mkIf (macvtapInterfaces != [ ]) {
macvtap-up = ''
set -eou pipefail
''
+ lib.concatMapStrings (
{
id,
mac,
macvtap,
...
}:
''
if [ -e /sys/class/net/${id} ]; then
${lib.getExe' pkgs.iproute2 "ip"} link delete '${id}'
fi
${lib.getExe' pkgs.iproute2 "ip"} link add link '${macvtap.link}' name '${id}' address '${mac}' type macvtap mode '${macvtap.mode}'
${lib.getExe' pkgs.iproute2 "ip"} link set '${id}' allmulticast on
if [ -f "/proc/sys/net/ipv6/conf/${id}/disable_ipv6" ]; then
echo 1 > "/proc/sys/net/ipv6/conf/${id}/disable_ipv6"
fi
${lib.getExe' pkgs.iproute2 "ip"} link set '${id}' up
${pkgs.coreutils-full}/bin/chown '${user}:${group}' /dev/tap$(< "/sys/class/net/${id}/ifindex")
''
) macvtapInterfaces;
macvtap-down = ''
set -ou pipefail
''
+ lib.concatMapStrings (
{ id, ... }:
''
${lib.getExe' pkgs.iproute2 "ip"} link delete '${id}'
''
) macvtapInterfaces;
})
];
}

207
example/microvm/mounts.nix Executable file
View file

@ -0,0 +1,207 @@
{ config, lib, ... }:
let
inherit (config.microvm) storeDiskType storeOnDisk writableStoreOverlay;
inherit
(import ../../lib {
inherit lib;
})
defaultFsType
withDriveLetters
;
hostStore = builtins.head (
builtins.filter ({ source, ... }: source == "/nix/store") config.microvm.shares
);
roStore = if storeOnDisk then "/nix/.ro-store" else hostStore.mountPoint;
roStoreDisk =
if storeOnDisk then
if
storeDiskType == "erofs"
# erofs supports filesystem labels
then
"/dev/disk/by-label/nix-store"
else
"/dev/vda"
else
throw "No disk letter when /nix/store is not in disk";
in
lib.mkIf config.microvm.guest.enable {
fileSystems = lib.mkMerge [
(
# built-in read-only store without overlay
lib.optionalAttrs (storeOnDisk && writableStoreOverlay == null) {
"/nix/store" = {
device = roStoreDisk;
fsType = storeDiskType;
options = [ "x-systemd.requires=systemd-modules-load.service" ];
neededForBoot = true;
noCheck = true;
};
}
)
(
# host store is mounted somewhere else,
# bind-mount to the proper place
lib.optionalAttrs
(
!storeOnDisk && config.microvm.writableStoreOverlay == null && hostStore.mountPoint != "/nix/store"
)
{
"/nix/store" = {
device = hostStore.mountPoint;
options = [ "bind" ];
neededForBoot = true;
};
}
)
(
# built-in read-only store for the overlay
lib.optionalAttrs (storeOnDisk && writableStoreOverlay != null) {
"/nix/.ro-store" = {
device = roStoreDisk;
fsType = storeDiskType;
options = [ "x-systemd.requires=systemd-modules-load.service" ];
neededForBoot = true;
noCheck = true;
};
}
)
(
# mount store with writable overlay
lib.optionalAttrs (writableStoreOverlay != null) {
"/nix/store" = {
device = "overlay";
fsType = "overlay";
neededForBoot = true;
options = [
"lowerdir=${roStore}"
"upperdir=${writableStoreOverlay}/store"
"workdir=${writableStoreOverlay}/work"
];
depends = [
roStore
writableStoreOverlay
];
};
}
)
{
# a tmpfs / by default. can be overwritten.
"/" = lib.mkDefault {
device = "rootfs";
fsType = "tmpfs";
options = [ "size=50%,mode=0755" ];
neededForBoot = true;
};
}
(
# Volumes
builtins.foldl' (
result:
{
label,
mountPoint,
letter,
fsType ? defaultFsType,
...
}:
result
// lib.optionalAttrs (mountPoint != null) {
"${mountPoint}" = {
inherit fsType;
# Prioritize identifying a device by label if provided. This
# minimizes the risk of misidentifying a device.
device = if label != null then "/dev/disk/by-label/${label}" else "/dev/vd${letter}";
}
// lib.optionalAttrs (mountPoint == config.microvm.writableStoreOverlay) {
neededForBoot = true;
};
}
) { } (withDriveLetters config.microvm)
)
(
# 9p/virtiofs Shares
builtins.foldl' (
result:
{
mountPoint,
tag,
proto,
source,
...
}:
result
// {
"${mountPoint}" = {
device = tag;
fsType = proto;
options =
{
"virtiofs" = [
"defaults"
"x-systemd.requires=systemd-modules-load.service"
];
"9p" = [
"trans=virtio"
"version=9p2000.L"
"msize=65536"
"x-systemd.requires=systemd-modules-load.service"
];
}
.${proto};
}
// lib.optionalAttrs (source == "/nix/store" || mountPoint == config.microvm.writableStoreOverlay) {
neededForBoot = true;
};
}
) { } config.microvm.shares
)
];
# boot.initrd.systemd patchups copied from <nixpkgs/nixos/modules/virtualisation/qemu-vm.nix>
boot.initrd.systemd = lib.mkIf (config.boot.initrd.systemd.enable && writableStoreOverlay != null) {
mounts = [
{
where = "/sysroot/nix/store";
what = "overlay";
type = "overlay";
options = builtins.concatStringsSep "," [
"lowerdir=/sysroot${roStore}"
"upperdir=/sysroot${writableStoreOverlay}/store"
"workdir=/sysroot${writableStoreOverlay}/work"
];
wantedBy = [ "initrd-fs.target" ];
before = [ "initrd-fs.target" ];
requires = [ "rw-store.service" ];
after = [ "rw-store.service" ];
unitConfig.RequiresMountsFor = "/sysroot/${roStore}";
}
];
services.rw-store = {
unitConfig = {
DefaultDependencies = false;
RequiresMountsFor = "/sysroot${writableStoreOverlay}";
};
serviceConfig = {
Type = "oneshot";
ExecStart = "/bin/mkdir -p -m 0755 /sysroot${writableStoreOverlay}/store /sysroot${writableStoreOverlay}/work /sysroot/nix/store";
};
};
};
# Fix for hanging shutdown
systemd.mounts = lib.mkIf config.boot.initrd.systemd.enable [
{
what = "store";
where = "/nix/store";
# Generate a `nix-store.mount.d/overrides.conf`
overrideStrategy = "asDropin";
unitConfig.DefaultDependencies = false;
}
];
}

65
example/microvm/optimization.nix Executable file
View file

@ -0,0 +1,65 @@
# Closure size and startup time optimization for disposable use-cases
{
config,
options,
lib,
...
}:
let
cfg = config.microvm;
canSwitchViaSsh =
config.services.openssh.enable
&&
# Is the /nix/store mounted from the host?
builtins.any ({ source, ... }: source == "/nix/store") config.microvm.shares;
in
lib.mkIf (cfg.guest.enable && cfg.optimize.enable) {
# The docs are pretty chonky
documentation.enable = lib.mkDefault false;
boot = {
initrd.systemd = {
# Use systemd initrd for startup speed.
# TODO: error mounting /nix/store on crosvm, kvmtool
enable = lib.mkDefault (
builtins.elem cfg.hypervisor [
"qemu"
"cloud-hypervisor"
"firecracker"
"stratovirt"
]
);
tpm2.enable = lib.mkDefault false;
};
kernelParams = [
# we only need one serial console
"8250.nr_uarts=1"
];
swraid.enable = lib.mkDefault false;
};
nixpkgs.overlays = [
(final: prev: {
stratovirt = prev.stratovirt.override { gtk3 = null; };
})
];
# networkd is used due to some strange startup time issues with nixos's
# homegrown dhcp implementation
networking.useNetworkd = lib.mkDefault true;
systemd = {
# Due to a bug in systemd-networkd: https://github.com/systemd/systemd/issues/29388
# we cannot use systemd-networkd-wait-online.
network.wait-online.enable = lib.mkDefault false;
tpm2.enable = lib.mkDefault false;
};
# Exclude switch-to-configuration.pl from toplevel.
system = lib.optionalAttrs (options.system ? switch && !canSwitchViaSsh) {
switch.enable = lib.mkDefault false;
};
}

806
example/microvm/options.nix Executable file
View file

@ -0,0 +1,806 @@
{
config,
lib,
pkgs,
...
}:
let
self-lib = import ../../lib {
inherit lib;
};
cfg = config.microvm;
hostName = config.networking.hostName or "$HOSTNAME";
kernelAtLeast = lib.versionAtLeast config.boot.kernelPackages.kernel.version;
in
{
options.microvm = with lib; {
guest.enable = mkOption {
type = types.bool;
default = true;
description = ''
Whether to enable the microvm.nix guest module at all.
'';
};
optimize.enable = lib.mkOption {
description = ''
Enables some optimizations by default to closure size and startup time:
- defaults documentation to off
- defaults to using systemd in initrd
- use systemd-networkd
- disables systemd-network-wait-online
- disables NixOS system switching if the host store is not mounted
This takes a few hundred MB off the closure size, including qemu,
allowing for putting MicroVMs inside Docker containers.
'';
type = lib.types.bool;
default = true;
};
cpu = mkOption {
type = with types; nullOr str;
default = null;
description = ''
What CPU to emulate, if any. If different from the host
architecture, it will have a serious performance hit.
::: {.note}
Only supported with qemu.
:::
'';
};
hypervisor = mkOption {
type = types.enum self-lib.hypervisors;
default = "qemu";
description = ''
Which hypervisor to use for this MicroVM
Choose one of: ${lib.concatStringsSep ", " self-lib.hypervisors}
'';
};
preStart = mkOption {
description = "Commands to run before starting the hypervisor";
default = "";
type = types.lines;
};
socket = mkOption {
description = "Hypervisor control socket path";
default = "${hostName}.sock";
defaultText = literalExpression ''"''${hostName}.sock"'';
type = with types; nullOr str;
};
user = mkOption {
description = "User to switch to when started as root";
default = null;
type = with types; nullOr str;
};
kernel = mkOption {
description = "Kernel package to use for MicroVM runners. Better set `boot.kernelPackages` instead.";
default = config.boot.kernelPackages.kernel;
defaultText = literalExpression ''"''${config.boot.kernelPackages.kernel}"'';
type = types.package;
};
initrdPath = mkOption {
description = "Path to the initrd file in the initrd package";
default = "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}";
defaultText = literalExpression ''"''${config.system.build.initialRamdisk}/''${config.system.boot.loader.initrdFile}"'';
type = types.path;
};
vcpu = mkOption {
description = "Number of virtual CPU cores";
default = 1;
type = types.ints.positive;
};
mem = mkOption {
description = "Amount of RAM in megabytes";
default = 512;
type = types.ints.positive;
};
hugepageMem = mkOption {
type = types.bool;
default = false;
description = ''
Whether to use hugepages as memory backend.
(Currently only respected if using cloud-hypervisor)
'';
};
hotplugMem = mkOption {
description = ''
Amount of hotplug memory in megabytes.
This describes the maximum amount of memory that can be dynamically added to the VM with virtio-mem.
'';
default = 0;
type = types.ints.unsigned;
};
hotpluggedMem = mkOption {
description = ''
Amount of hotplugged memory in megabytes.
This basically describes the amount of hotplug memory the VM starts with.
'';
default = config.microvm.hotplugMem;
type = types.ints.unsigned;
};
balloon = mkOption {
description = ''
Whether to enable ballooning.
By "inflating" or increasing the balloon the host can reduce the VMs
memory amount and reclaim it for itself.
When "deflating" or decreasing the balloon the host can give the memory
back to the VM.
virtio-mem is recommended over ballooning if supported by the hypervisor.
'';
default = false;
type = types.bool;
};
initialBalloonMem = mkOption {
description = ''
Amount of initial balloon memory in megabytes.
'';
default = 0;
type = types.ints.unsigned;
};
deflateOnOOM = mkOption {
type = types.bool;
default = true;
description = ''
Whether to enable automatic balloon deflation on out-of-memory.
'';
};
forwardPorts = mkOption {
type = types.listOf (
types.submodule {
options.from = mkOption {
type = types.enum [
"host"
"guest"
];
default = "host";
description = ''
Controls the direction in which the ports are mapped:
- <literal>"host"</literal> means traffic from the host ports
is forwarded to the given guest port.
- <literal>"guest"</literal> means traffic from the guest ports
is forwarded to the given host port.
'';
};
options.proto = mkOption {
type = types.enum [
"tcp"
"udp"
];
default = "tcp";
description = "The protocol to forward.";
};
options.host.address = mkOption {
type = types.str;
default = "";
description = "The IPv4 address of the host.";
};
options.host.port = mkOption {
type = types.port;
description = "The host port to be mapped.";
};
options.guest.address = mkOption {
type = types.str;
default = "";
description = "The IPv4 address on the guest VLAN.";
};
options.guest.port = mkOption {
type = types.port;
description = "The guest port to be mapped.";
};
}
);
default = [ ];
example = lib.literalExpression /* nix */ ''
[ # forward local port 2222 -> 22, to ssh into the VM
{ from = "host"; host.port = 2222; guest.port = 22; }
# forward local port 80 -> 10.0.2.10:80 in the VLAN
{ from = "guest";
guest.address = "10.0.2.10"; guest.port = 80;
host.address = "127.0.0.1"; host.port = 80;
}
]
'';
description = ''
When using the SLiRP user networking (default), this option allows to
forward ports to/from the host/guest.
::: {.warning}
If the NixOS firewall on the virtual machine is enabled, you
also have to open the guest ports to enable the traffic
between host and guest.
:::
::: {.note}
Currently QEMU supports only IPv4 forwarding.
:::
'';
};
volumes = mkOption {
description = "Disk images";
default = [ ];
type =
with types;
listOf (submodule {
options = {
image = mkOption {
type = str;
description = "Path to disk image on the host";
};
serial = mkOption {
type = nullOr str;
default = null;
description = "User-configured serial number for the disk";
};
direct = mkOption {
type = bool;
default = false;
description = "Whether to set O_DIRECT on the disk.";
};
readOnly = mkOption {
type = bool;
default = false;
description = "Turn off write access";
};
label = mkOption {
type = nullOr str;
default = null;
description = "Label of the volume, if any. Only applicable if `autoCreate` is true; otherwise labeling of the volume must be done manually";
};
mountPoint = mkOption {
type = nullOr path;
description = "If and where to mount the volume inside the container";
};
size = mkOption {
type = int;
description = "Volume size (in MiB) if created automatically";
};
autoCreate = mkOption {
type = bool;
default = true;
description = "Created image on host automatically before start?";
};
mkfsExtraArgs = mkOption {
type = listOf str;
default = [ ];
description = "Set extra Filesystem creation parameters";
};
fsType = mkOption {
type = str;
default = "ext4";
description = "Filesystem for automatic creation and mounting";
};
};
});
};
interfaces = mkOption {
description = "Network interfaces";
default = [ ];
type =
with types;
listOf (submodule {
options = {
type = mkOption {
type = enum [
"user"
"tap"
"macvtap"
"bridge"
];
description = ''
Interface type
'';
};
id = mkOption {
type = str;
description = ''
Interface name on the host
'';
};
macvtap.link = mkOption {
type = str;
description = ''
Attach network interface to host interface for type = "macvlan"
'';
};
macvtap.mode = mkOption {
type = enum [
"private"
"vepa"
"bridge"
"passthru"
"source"
];
description = ''
The MACVLAN mode to use
'';
};
bridge = mkOption {
type = nullOr str;
default = null;
description = ''
Attach network interface to host bridge interface for type = "bridge"
'';
};
mac = mkOption {
type = str;
description = ''
MAC address of the guest's network interface
'';
};
};
});
};
shares = mkOption {
description = "Shared directory trees";
default = [ ];
type =
with types;
listOf (
submodule (
{ config, ... }:
{
options = {
tag = mkOption {
type = str;
description = "Unique virtiofs daemon tag";
};
socket = mkOption {
type = nullOr str;
default = if config.proto == "virtiofs" then "${hostName}-virtiofs-${config.tag}.sock" else null;
description = "Socket for communication with virtiofs daemon";
};
source = mkOption {
type = nonEmptyStr;
description = "Path to shared directory tree";
};
securityModel = mkOption {
type = enum [
"passthrough"
"none"
"mapped"
"mapped-file"
];
default = "none";
description = "What security model to use for the shared directory";
};
mountPoint = mkOption {
type = path;
description = "Where to mount the share inside the container";
};
proto = mkOption {
type = enum [
"9p"
"virtiofs"
];
description = "Protocol for this share";
default = "9p";
};
readOnly = mkOption {
type = bool;
description = "Turn off write access";
default = false;
};
};
}
)
);
};
devices = mkOption {
description = "PCI/USB devices that are passed from the host to the MicroVM";
default = [ ];
example = literalExpression /* nix */ ''
[ {
bus = "pci";
path = "0000:01:00.0";
} {
bus = "pci";
path = "0000:01:01.0";
deviceExtraArgs = "id=hostId,x-igd-opregion=on";
} {
# QEMU only
bus = "usb";
path = "vendorid=0xabcd,productid=0x0123";
} ]
'';
type =
with types;
listOf (submodule {
options = {
bus = mkOption {
type = enum [
"pci"
"usb"
];
description = ''
Device is either on the `pci` or the `usb` bus
'';
};
path = mkOption {
type = str;
description = ''
Identification of the device on its bus
'';
};
qemu.deviceExtraArgs = mkOption {
type = with types; nullOr str;
default = null;
description = ''
Device additional arguments (optional)
'';
};
};
});
};
vsock.cid = mkOption {
default = null;
type = with types; nullOr int;
description = ''
Virtual Machine address;
setting it enables AF_VSOCK
The following are reserved:
- 0: Hypervisor
- 1: Loopback
- 2: Host
'';
};
kernelParams = mkOption {
type = with types; listOf str;
description = "Includes boot.kernelParams but doesn't end up in toplevel, thereby allowing references to toplevel";
};
storeOnDisk = mkOption {
type = types.bool;
default = !lib.any ({ source, ... }: source == "/nix/store") config.microvm.shares;
description = "Whether to boot with the storeDisk, that is, unless the host's /nix/store is a microvm.share.";
};
registerClosure =
lib.mkEnableOption ''
Register system closure's store paths in Nix db.
While enabled by default, this option may be incompatible with a persistent writable store overlay.
''
// {
default = config.microvm.guest.enable;
};
writableStoreOverlay = mkOption {
type = with types; nullOr str;
default = null;
example = "/nix/.rw-store";
description = ''
Path to the writable /nix/store overlay.
If set to a filesystem path, the initrd will mount /nix/store
as an overlay filesystem consisting of the read-only part as a
host share or from the built storeDisk, and this configuration
option as the writable overlay part. This allows you to build
nix derivations *inside* the VM.
Make sure that the path points to a writable filesystem
(tmpfs, volume, or share).
'';
};
graphics = {
enable = mkOption {
type = types.bool;
default = false;
description = ''
Enable GUI support.
MicroVMs with graphics are intended for the interactive
use-case. They cannot be started through systemd jobs.
The display backend is chosen by `microvm.graphics.backend`.
'';
};
backend = mkOption {
type = types.enum [
"gtk"
"cocoa"
];
default = if pkgs.stdenv.hostPlatform.isDarwin then "cocoa" else "gtk";
defaultText = lib.literalExpression ''if pkgs.stdenv.hostPlatform.isDarwin then "cocoa" else "gtk"'';
description = ''
QEMU display backend to use when `graphics.enable` is true.
Defaults to `cocoa` on Darwin hosts and `gtk` otherwise.
'';
};
socket = mkOption {
type = types.str;
default = "${hostName}-gpu.sock";
description = ''
Path of vhost-user socket
'';
};
};
vmHostPackages = mkOption {
description = "If set, overrides the default host package.";
example = "nixpkgs.legacyPackages.aarch64-darwin.pkgs";
type = types.nullOr types.pkgs;
default = if cfg.cpu == null then pkgs else pkgs.buildPackages;
defaultText = lib.literalExpression "if config.microvm.cpu == null then pkgs else pkgs.buildPackages";
};
qemu.machine = mkOption {
type = types.str;
description = ''
QEMU machine model, eg. `microvm`, or `q35`
Get a full list with `qemu-system-x86_64 -M help`
This has a default declared with `lib.mkDefault` because it
depends on ''${pkgs.system}.
'';
};
qemu.machineOpts = mkOption {
type = with types; nullOr (attrsOf str);
default = null;
description = "Overwrite the default machine model options.";
};
qemu.extraArgs = mkOption {
type = with types; listOf str;
default = [ ];
description = "Extra arguments to pass to qemu.";
};
qemu.serialConsole = mkOption {
type = types.bool;
default = true;
description = ''
Whether to enable the virtual serial console on qemu.
'';
};
cloud-hypervisor.platformOEMStrings = mkOption {
type = with types; listOf str;
default = [ ];
description = ''
Extra arguments to pass to cloud-hypervisor's --platform oem_strings=[] argument.
All the oem strings will be concatenated with a comma (,) and wrapped in oem_string=[].
Do not include oem_string= or the [] brackets in the value.
The resulting string will be combined with any --platform options in
`config.microvm.cloud-hypervisor.extraArgs` and passed as a single
--platform option to cloud-hypervisor
'';
example = lib.literalExpression /* nix */ ''[ "io.systemd.credential:APIKEY=supersecret" ]'';
};
cloud-hypervisor.extraArgs = mkOption {
type = with types; listOf str;
default = [ ];
description = "Extra arguments to pass to cloud-hypervisor.";
};
crosvm.extraArgs = mkOption {
type = with types; listOf str;
default = [ ];
description = "Extra arguments to pass to crosvm.";
};
crosvm.pivotRoot = mkOption {
type = with types; nullOr str;
default = null;
description = "A Hypervisor's sandbox directory";
};
firecracker.cpu = mkOption {
type = with types; nullOr attrs;
default = null;
description = "Custom CPU template passed to firecracker.";
};
prettyProcnames = mkOption {
type = types.bool;
default = true;
description = ''
Set a recognizable process name right before executing the Hyperisor.
'';
};
virtiofsd.inodeFileHandles = mkOption {
type =
with types;
nullOr (enum [
"never"
"prefer"
"mandatory"
]);
default = null;
description = ''
When to use file handles to reference inodes instead of O_PATH file descriptors
(never, prefer, mandatory)
Allows you to overwrite default behavior in case you hit "too
many open files" on eg. ZFS.
<https://gitlab.com/virtio-fs/virtiofsd/-/issues/121>
'';
};
virtiofsd.threadPoolSize = mkOption {
type =
with types;
oneOf [
str
ints.unsigned
];
default = "`nproc`";
description = ''
The amounts of threads virtiofsd should spawn. This option also takes the special
string `\`nproc\`` which spawns as many threads as the host has cores.
'';
};
virtiofsd.group = mkOption {
type = with types; nullOr str;
default = "kvm";
description = ''
The name of the group that will own the Unix domain socket file that virtiofsd creates for communication with the hypervisor.
If null, the socket will have group ownership of the user running the hypervisor.
'';
};
virtiofsd.extraArgs = mkOption {
type = with types; listOf str;
default = [ ];
description = ''
Extra command-line switch to pass to virtiofsd.
'';
};
runner = mkOption {
description = "Generated Hypervisor runner for this NixOS";
type = with types; attrsOf package;
};
declaredRunner = mkOption {
description = "Generated Hypervisor declared by `config.microvm.hypervisor`";
type = types.package;
default = config.microvm.runner.${config.microvm.hypervisor};
defaultText = literalExpression ''"config.microvm.runner.''${config.microvm.hypervisor}"'';
};
binScripts = mkOption {
description = ''
Script snippets that end up in the runner package's bin/ directory
'';
default = { };
type = with types; attrsOf lines;
};
storeDiskType = mkOption {
type = types.enum [
"squashfs"
"erofs"
];
description = ''
Boot disk file system type: squashfs is smaller, erofs is supposed to be faster.
Defaults to erofs, unless the NixOS hardened profile is detected.
'';
};
storeDiskErofsFlags = mkOption {
type = with types; listOf str;
description = ''
Flags to pass to mkfs.erofs
Omit `"-Efragments"` and `"-Ededupe"` to enable multi-threading.
'';
default = [
"-zlz4hc"
]
++ lib.optional (kernelAtLeast "5.16") "-Eztailpacking"
++ lib.optionals (kernelAtLeast "6.1") [
# not implemented with multi-threading
"-Efragments"
"-Ededupe"
];
defaultText = lib.literalExpression ''
[ "-zlz4hc" ]
++ lib.optional (kernelAtLeast "5.16") "-Eztailpacking"
++ lib.optionals (kernelAtLeast "6.1") [
"-Efragments"
"-Ededupe"
]
'';
};
storeDiskSquashfsFlags = mkOption {
type = with types; listOf str;
description = "Flags to pass to gensquashfs";
default = [
"-c"
"zstd"
"-j"
"$NIX_BUILD_CORES"
];
};
systemSymlink = mkOption {
type = types.bool;
default = !config.microvm.storeOnDisk;
description = ''
Whether to inclcude a symlink of `config.system.build.toplevel` to `share/microvm/system`.
This is required for commands like `microvm -l` to function but removes reference to the uncompressed store content when using a disk image for the nix store.
'';
};
credentialFiles = mkOption {
type = with types; attrsOf path;
default = { };
description = ''
Key-value pairs of credential files that will be loaded into the vm using systemd's io.systemd.credential feature.
'';
example = literalExpression /* nix */ ''
{
SOPS_AGE_KEY = "/run/secrets/guest_microvm_age_key";
}
'';
};
};
imports = [
(lib.mkRemovedOptionModule [
"microvm"
"balloonMem"
] "The balloonMem option has been removed and replaced by the boolean option balloon")
];
config = lib.mkMerge [
{
microvm.qemu.machine = lib.mkIf (pkgs.stdenv.hostPlatform.system == "x86_64-linux") (
lib.mkDefault "microvm"
);
}
{
microvm.qemu.machine = lib.mkIf (pkgs.stdenv.hostPlatform.system == "aarch64-linux") (
lib.mkDefault "virt"
);
}
];
}

47
example/microvm/pci-devices.nix Executable file
View file

@ -0,0 +1,47 @@
{
config,
lib,
pkgs,
...
}:
let
pciDevices = builtins.filter ({ bus, ... }: bus == "pci") config.microvm.devices;
# TODO: don't hardcode but obtain from host config
user = "microvm";
group = "kvm";
in
{
microvm.binScripts.pci-setup = lib.mkIf (pciDevices != [ ]) (
''
set -eou pipefail
${pkgs.kmod}/bin/modprobe vfio-pci
''
+ lib.concatMapStrings (
{ path, ... }:
''
cd /sys/bus/pci/devices/${path}
if [ -e driver ]; then
echo ${path} > driver/unbind
fi
echo vfio-pci > driver_override
echo ${path} > /sys/bus/pci/drivers_probe
''
+
# In order to access the vfio dev the permissions must be set
# for the user/group running the VMM later.
#
# Insprired by https://www.kernel.org/doc/html/next/driver-api/vfio.html#vfio-usage-example
#
# assert we could get the IOMMU group number (=: name of VFIO dev)
''
[[ -e iommu_group ]] || exit 1
VFIO_DEV=$(basename $(readlink iommu_group))
echo "Making VFIO device $VFIO_DEV accessible for user"
chown ${user}:${group} /dev/vfio/$VFIO_DEV
''
) pciDevices
);
}

252
example/microvm/ssh-deploy.nix Executable file
View file

@ -0,0 +1,252 @@
{
config,
lib,
pkgs,
...
}:
let
hostName = config.networking.hostName or "$HOSTNAME";
inherit (config.system.build) toplevel;
inherit (config.microvm) declaredRunner;
inherit (config) nix;
closureInfo = pkgs.closureInfo {
rootPaths = [ config.system.build.toplevel ];
};
# Don't build these but get the derivation paths for building on a
# remote host, and for switching via SSH.
paths = builtins.mapAttrs (_: builtins.unsafeDiscardStringContext) {
closureInfoOut = closureInfo.outPath;
closureInfoDrv = closureInfo.drvPath;
toplevelOut = toplevel.outPath;
toplevelDrv = toplevel.drvPath;
nixOut = nix.package.outPath;
nixDrv = nix.package.drvPath;
runnerDrv = declaredRunner.drvPath;
};
canSwitchViaSsh =
config.system.switch.enable
&&
# MicroVM must be reachable through SSH
config.services.openssh.enable
&&
# Is the /nix/store mounted from the host?
builtins.any ({ source, ... }: source == "/nix/store") config.microvm.shares;
in
{
# Declarations with documentation
options.microvm.deploy = {
installOnHost = lib.mkOption {
description = ''
Use this script to deploy the working state of your local
Flake on a target host that imports
`microvm.nixosModules.host`:
```
nix run .#nixosConfigurations.${hostName}.config.microvm.deploy.installOnHost root@example.com
ssh root@example.com systemctl restart microvm@${hostName}
```
- Evaluate this MicroVM to a derivation
- Copy the derivation to the target host
- Build the MicroVM runner on the target host
- Install/update the MicroVM on the target host
Can be followed by either:
- `systemctl restart microvm@${hostName}.service` on the
target host, or
- `config.microvm.deploy.sshSwitch`
'';
type = lib.types.package;
};
sshSwitch = lib.mkOption {
description = ''
Instead of restarting a MicroVM for an update, perform it via
SSH.
The host's /nix/store must be mounted, and the built
`config.microvm.declaredRunner` must exist in it. Use
`microvm.deploy.installOnHost` like this:
```
nix run .#nixosConfigurations.${hostName}.config.microvm.deploy.installOnHost root@example.com
nix run .#nixosConfigurations.${hostName}.config.microvm.deploy.sshSwitch root@my-microvm.example.com switch
```
'';
type = with lib.types; nullOr package;
default = null;
};
rebuild = lib.mkOption {
description = ''
`config.microvm.deploy.installOnHost` and `.sshSwitch` in one
script. Akin to what nixos-rebuild does but for a remote
MicroVM.
```
nix run .#nixosConfigurations.${hostName}.config.microvm.deploy.rebuild root@example.com root@my-microvm.example.com switch
```
'';
type = with lib.types; nullOr package;
default = null;
};
};
# Implementations
config.microvm.deploy = {
installOnHost = pkgs.writeShellScriptBin "microvm-install-on-host" ''
set -eou pipefail
USAGE="Usage: $0 root@<host> [--use-remote-sudo]"
HOST="$1"
if [[ -z "$HOST" ]]; then
echo $USAGE
exit 1
fi
shift
SSH_CMD="bash"
if [ $# -gt 0 ]; then
if [ "$1" == "--use-remote-sudo" ]; then
SSH_CMD="sudo bash"
shift
else
echo "$USAGE"
exit 1
fi
fi
echo "Copying derivations to $HOST"
nix copy --no-check-sigs --to "ssh-ng://$HOST" \
--derivation \
"${paths.closureInfoDrv}^out" \
"${paths.runnerDrv}^out"
ssh "$HOST" -- $SSH_CMD -e <<__SSH__
set -eou pipefail
echo "Initializing MicroVM ${hostName} if necessary"
mkdir -p /nix/var/nix/gcroots/microvm
mkdir -p /var/lib/microvms/${hostName}
cd /var/lib/microvms/${hostName}
chown microvm:kvm .
chmod 0755 .
ln -sfT \$PWD/current /nix/var/nix/gcroots/microvm/${hostName}
ln -sfT \$PWD/booted /nix/var/nix/gcroots/microvm/booted-${hostName}
ln -sfT \$PWD/old /nix/var/nix/gcroots/microvm/old-${hostName}
echo "Building toplevel ${paths.toplevelOut}"
nix build -L --accept-flake-config --no-link \
${
with paths;
lib.concatMapStringsSep " " (drv: "'${drv}^out'") [
nixDrv
closureInfoDrv
toplevelDrv
]
}
echo "Building MicroVM runner for ${hostName}"
nix build -L --accept-flake-config -o new \
"${paths.runnerDrv}^out"
if [[ $(realpath ./current) != $(realpath ./new) ]]; then
echo "Installing MicroVM ${hostName}"
rm -f old
if [ -e current ]; then
mv current old
fi
mv new current
if [ -e old ]; then
echo "Success. Diff:"
nix --extra-experimental-features nix-command \
store diff-closures ./old ./current \
|| true
else
echo "Success."
fi
else
echo "MicroVM ${hostName} is already installed"
fi
__SSH__
'';
sshSwitch = lib.mkIf canSwitchViaSsh (
pkgs.writeShellScriptBin "microvm-switch" ''
set -eou pipefail
USAGE="Usage: $0 root@<target> [--use-remote-sudo]"
TARGET="$1"
if [[ -z "$TARGET" ]]; then
echo "$USAGE"
exit 1
fi
shift
SSH_CMD="bash"
if [ $# -gt 0 ]; then
if [ "$1" == "--use-remote-sudo" ]; then
SSH_CMD="sudo bash"
shift
else
echo "$USAGE"
exit 1
fi
fi
ssh "$TARGET" $SSH_CMD -e <<__SSH__
set -eou pipefail
hostname=\$(cat /etc/hostname)
if [[ "\$hostname" != "${hostName}" ]]; then
echo "Attempting to deploy NixOS ${hostName} on host \$hostname"
exit 1
fi
# refresh nix db which is required for nix-env -p ... --set
echo "Refreshing Nix database"
${paths.nixOut}/bin/nix-store --load-db < ${paths.closureInfoOut}/registration
${paths.nixOut}/bin/nix-env -p /nix/var/nix/profiles/system --set ${paths.toplevelOut}
${paths.toplevelOut}/bin/switch-to-configuration "''${@:-switch}"
__SSH__
''
);
rebuild =
with config.microvm.deploy;
pkgs.writeShellScriptBin "microvm-rebuild" ''
set -eou pipefail
HOST="$1"
shift
TARGET="$1"
shift
OPTS="$@"
if [ $# -gt 0 ]; then
if [ "$1" == "--use-remote-sudo" ]; then
OPTS="$1"
shift
fi
fi
if [[ -z "$HOST" || -z "$TARGET" || $# -gt 0 ]]; then
echo "Usage: $0 root@<host> root@<target> [--use-remote-sudo] switch"
exit 1
fi
${lib.getExe installOnHost} "$HOST" $OPTS
${
if canSwitchViaSsh then
''${lib.getExe sshSwitch} "$TARGET" $OPTS''
else
''ssh "$HOST" -- systemctl restart "microvm@${hostName}.service"''
}
'';
};
}

124
example/microvm/store-disk.nix Executable file
View file

@ -0,0 +1,124 @@
{
config,
lib,
pkgs,
...
}:
let
regInfo = pkgs.closureInfo {
rootPaths = [ config.system.build.toplevel ];
};
erofs-utils =
# Are any extended options specified?
if
lib.any (
with lib;
flip elem [
"-Ededupe"
"-Efragments"
]
) config.microvm.storeDiskErofsFlags
then
# If extended options are present,
# stick to the single-threaded erofs-utils
# to not scare anyone with warning messages.
pkgs.buildPackages.erofs-utils
else
# If no extended options are configured,
# rebuild mkfs.erofs with multi-threading.
pkgs.buildPackages.erofs-utils.overrideAttrs (attrs: {
configureFlags = attrs.configureFlags ++ [
"--enable-multithreading"
];
});
erofsFlags = builtins.concatStringsSep " " config.microvm.storeDiskErofsFlags;
squashfsFlags = builtins.concatStringsSep " " config.microvm.storeDiskSquashfsFlags;
mkfsCommand =
{
squashfs = "gensquashfs ${squashfsFlags} -D store --all-root -q $out";
erofs = "mkfs.erofs ${erofsFlags} -T 0 --all-root -L nix-store --mount-point=/nix/store $out store";
}
.${config.microvm.storeDiskType};
writeClosure = pkgs.writeClosure or pkgs.writeReferencesToFile;
storeDiskContents = writeClosure (
[ config.system.build.toplevel ] ++ lib.optional config.nix.enable regInfo
);
in
{
options.microvm.storeDisk =
with lib;
mkOption {
type = types.path;
description = ''
Generated
'';
};
config = lib.mkMerge [
(lib.mkIf (config.microvm.guest.enable && config.microvm.storeOnDisk) {
# nixos/modules/profiles/hardened.nix forbids erofs.
# HACK: Other NixOS modules populate
# config.boot.blacklistedKernelModules depending on the boot
# filesystems, so checking on that directly would result in an
# infinite recursion.
microvm.storeDiskType = lib.mkDefault (
if config.security.virtualisation.flushL1DataCache == "always" then "squashfs" else "erofs"
);
boot.initrd.availableKernelModules = [
config.microvm.storeDiskType
];
microvm.storeDisk =
pkgs.runCommandLocal "microvm-store-disk.${config.microvm.storeDiskType}"
{
nativeBuildInputs = [
pkgs.buildPackages.time
pkgs.buildPackages.bubblewrap
{
squashfs = pkgs.buildPackages.squashfs-tools-ng;
erofs = erofs-utils;
}
.${config.microvm.storeDiskType}
];
passthru = {
inherit regInfo;
};
__structuredAttrs = true;
unsafeDiscardReferences.out = true;
}
''
mkdir store
BWRAP_ARGS="--dev-bind / / --chdir $(pwd)"
for d in $(sort -u ${storeDiskContents}); do
BWRAP_ARGS="$BWRAP_ARGS --ro-bind $d $(pwd)/store/$(basename $d)"
done
echo Creating a ${config.microvm.storeDiskType}
bwrap $BWRAP_ARGS -- time ${mkfsCommand} || \
(
echo "Bubblewrap failed. Falling back to copying...">&2
cp -a $(sort -u ${storeDiskContents}) store/
time ${mkfsCommand}
)
'';
})
(lib.mkIf (config.microvm.registerClosure && config.nix.enable) {
microvm.kernelParams = [
"regInfo=${regInfo}/registration"
];
boot.postBootCommands = ''
if [[ "$(cat /proc/cmdline)" =~ regInfo=([^ ]*) ]]; then
${config.nix.package.out}/bin/nix-store --load-db < ''${BASH_REMATCH[1]}
fi
'';
})
];
}

82
example/microvm/system.nix Executable file
View file

@ -0,0 +1,82 @@
{
pkgs,
lib,
config,
...
}:
{
config = lib.mkIf config.microvm.guest.enable {
assertions = [
{
assertion =
(config.microvm.writableStoreOverlay != null)
-> (!config.nix.optimise.automatic && !config.nix.settings.auto-optimise-store);
message = ''
`nix.optimise.automatic` and `nix.settings.auto-optimise-store` do not work with `microvm.writableStoreOverlay`.
'';
}
];
boot.loader.grub.enable = false;
# boot.initrd.systemd.enable = lib.mkDefault true;
boot.initrd.kernelModules = [
"virtio_mmio"
"virtio_pci"
"virtio_blk"
"9pnet_virtio"
"9p"
"virtiofs"
]
++
lib.optionals
(pkgs.stdenv.targetPlatform.system == "x86_64-linux" && config.microvm.hypervisor == "firecracker")
[
# Keyboard controller that can receive CtrlAltDel
"i8042"
]
++ lib.optionals (config.microvm.writableStoreOverlay != null) [
"overlay"
];
microvm.kernelParams =
let
# When a store disk is used, we can drop references to the packed contents as the squashfs/erofs contains all paths.
toplevel =
if config.microvm.storeOnDisk then
builtins.unsafeDiscardStringContext config.system.build.toplevel
else
config.system.build.toplevel;
in
config.boot.kernelParams
++ [
"init=${toplevel}/init"
];
# modules that consume boot time but have rare use-cases
boot.blacklistedKernelModules = [
"rfkill"
"intel_pstate"
]
++ lib.optional (!config.microvm.graphics.enable) "drm";
systemd =
let
# nix-daemon works only with a writable /nix/store
enableNixDaemon = config.microvm.writableStoreOverlay != null;
in
{
services.nix-daemon.enable = lib.mkDefault enableNixDaemon;
sockets.nix-daemon.enable = lib.mkDefault enableNixDaemon;
# consumes a lot of boot time
services.mount-pstore.enable = false;
# just fails in the usual usage of microvm.nix
generators = {
systemd-gpt-auto-generator = "/dev/null";
};
};
};
}

93
example/microvm/virtiofsd/default.nix Executable file
View file

@ -0,0 +1,93 @@
{
config,
lib,
pkgs,
...
}:
let
virtiofsShares = builtins.filter ({ proto, ... }: proto == "virtiofs") config.microvm.shares;
requiresVirtiofsd = virtiofsShares != [ ];
inherit (pkgs.python3Packages) supervisor;
supervisord = lib.getExe' supervisor "supervisord";
supervisorctl = lib.getExe' supervisor "supervisorctl";
in
{
microvm.binScripts = lib.mkIf requiresVirtiofsd {
virtiofsd-run =
let
supervisordConfig = {
supervisord.nodaemon = true;
"eventlistener:notify" = {
command = pkgs.writers.writePython3 "supervisord-event-handler" { } (
pkgs.replaceVars ./supervisord-event-handler.py {
# 1 for the event handler process
virtiofsdCount = 1 + builtins.length virtiofsShares;
}
);
events = "PROCESS_STATE";
};
}
// builtins.listToAttrs (
map (
{
tag,
socket,
source,
readOnly,
...
}:
{
name = "program:virtiofsd-${tag}";
value = {
stderr_syslog = true;
stdout_syslog = true;
autorestart = true;
command = pkgs.writeShellScript "virtiofsd-${tag}" ''
if [ $(id -u) = 0 ]; then
OPT_RLIMIT="--rlimit-nofile 1048576"
else
OPT_RLIMIT=""
fi
exec ${lib.getExe pkgs.virtiofsd} \
--socket-path=${lib.escapeShellArg socket} \
${
lib.optionalString (
config.microvm.virtiofsd.group != null
) "--socket-group=${config.microvm.virtiofsd.group}"
} \
--shared-dir=${lib.escapeShellArg source} \
$OPT_RLIMIT \
--thread-pool-size ${toString config.microvm.virtiofsd.threadPoolSize} \
--posix-acl --xattr \
${
lib.optionalString (
config.microvm.virtiofsd.inodeFileHandles != null
) "--inode-file-handles=${config.microvm.virtiofsd.inodeFileHandles}"
} \
${lib.optionalString (config.microvm.hypervisor == "crosvm") "--tag=${tag}"} \
${lib.optionalString readOnly "--readonly"} \
${lib.concatStringsSep " " config.microvm.virtiofsd.extraArgs}
'';
};
}
) virtiofsShares
);
supervisordConfigFile = pkgs.writeText "${config.networking.hostName}-virtiofsd-supervisord.conf" (
lib.generators.toINI { } supervisordConfig
);
in
''
exec ${supervisord} --configuration ${supervisordConfigFile}
'';
virtiofsd-shutdown = ''
exec ${supervisorctl} stop
'';
};
}

44
example/microvm/virtiofsd/supervisord-event-handler.py Executable file
View file

@ -0,0 +1,44 @@
import subprocess
import sys
def write_stdout(s):
# only eventlistener protocol messages may be sent to stdout
sys.stdout.write(s)
sys.stdout.flush()
def write_stderr(s):
sys.stderr.write(s)
sys.stderr.flush()
def main():
count = 0
expected_count = @virtiofsdCount@
while True:
write_stdout('READY\n')
line = sys.stdin.readline()
# read event payload and print it to stderr
headers = dict([x.split(':') for x in line.split()])
sys.stdin.read(int(headers['len']))
# body = dict([x.split(':') for x in data.split()])
if headers["eventname"] == "PROCESS_STATE_RUNNING":
count += 1
write_stderr("Process state running...\n")
if headers["eventname"] == "PROCESS_STATE_STOPPING":
count -= 1
write_stderr("Process state stopping...\n")
if count >= expected_count:
subprocess.run(["systemd-notify", "--ready"])
write_stdout('RESULT 2\nOK')
if __name__ == '__main__':
main()