From 097a85cde208b4e3a27a8122bab22586f9d895d5 Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 21 Nov 2025 18:29:53 -0600 Subject: [PATCH] feat: added fedifetcher --- modules/nixos/guests/defenseio/default.nix | 12 +- modules/nixos/guests/mastodon/default.nix | 179 ++++++++++++++------- 2 files changed, 128 insertions(+), 63 deletions(-) diff --git a/modules/nixos/guests/defenseio/default.nix b/modules/nixos/guests/defenseio/default.nix index e6f7f67..e56ae26 100755 --- a/modules/nixos/guests/defenseio/default.nix +++ b/modules/nixos/guests/defenseio/default.nix @@ -75,12 +75,12 @@ in ]; mem = let - num = 1024; - ceresRam = num * 50; - erisRam = num * 7; - marsRam = num * 22; - deimosRam = num * 7; - phobosRam = num * 7; + num = num: (num * 1024); + ceresRam = num 50; + erisRam = num 7; + marsRam = num 22; + deimosRam = num 7; + phobosRam = num 7; in deviceLogic ceresRam erisRam marsRam deimosRam phobosRam; shares = [ diff --git a/modules/nixos/guests/mastodon/default.nix b/modules/nixos/guests/mastodon/default.nix index 5158ac3..dbd2158 100755 --- a/modules/nixos/guests/mastodon/default.nix +++ b/modules/nixos/guests/mastodon/default.nix @@ -14,32 +14,49 @@ let host = serviceCfg.domains.url0; dns0 = instances.web.dns.provider0; dns0Path = "dns/${dns0}"; + + fedifetcherConfig = pkgs.writeText "fedifetcher-config.json" ( + builtins.toJSON { + server = "https://${host}"; + home-timeline-length = 200; + max-followings = 80; + from-notifications = 1; + max-bookmarks = 80; + max-favourites = 40; + backfill-with-context = 1; + backfill-mentioned-users = 1; + remember-users-for-hours = 168; + remember-hosts-for-days = 30; + http-timeout = 5; + lock-hours = 24; + log-level = "INFO"; + } + ); + + fedifetcherPython = pkgs.python3.withPackages ( + ps: with ps; [ + requests + pytz + beautifulsoup4 + ] + ); + + fedifetcherSrc = pkgs.fetchFromGitHub { + owner = "nanos"; + repo = "FediFetcher"; + rev = "main"; + hash = "sha256-/J7psV/mA7okuuO7/aXVVWS9p63eMncG2CEEGN38ip0="; + }; in { # If you need to start fresh for some reason, run these to create the new Admin account: # sudo -u mastodon mastodon-tootctl accounts create nick --email=nick@localhost --confirmed --role=Owner # sudo -u mastodon mastodon-tootctl accounts approve nick - # If you fuck up and lose the password, use this: # sudo mastodon-tootctl accounts modify --reset-password nick - # If you really fuck up and name yourself wrong, use this shit # sudo mastodon-tootctl accounts modify username --remove-role - # nixpkgs.overlays = [ - # ( - # final: prev: { - # mastodon = prev.mastodon.overrideAttrs (oldAttrs: { - # postPatch = - # (oldAttrs.postPatch or "") - # + '' - # patch -p1 < ${./chars.patch} - # ''; - # }); - # } - # ) - # ]; - microvm.vms = { ${serviceCfg.name} = { autostart = true; @@ -48,6 +65,7 @@ in system.stateVersion = "24.05"; time.timeZone = "America/Winnipeg"; users.users.root.openssh.authorizedKeys.keys = flake.config.people.users.${user0}.sshKeys; + services = { ${serviceCfg.name} = { enable = true; @@ -81,20 +99,15 @@ in sidekiqThreads = 25; sidekiqProcesses = { all = { - jobClasses = [ - ]; + jobClasses = [ ]; threads = null; }; default = { - jobClasses = [ - "default" - ]; + jobClasses = [ "default" ]; threads = 5; }; ingress = { - jobClasses = [ - "ingress" - ]; + jobClasses = [ "ingress" ]; threads = 5; }; push-pull = { @@ -105,9 +118,7 @@ in threads = 5; }; mailers = { - jobClasses = [ - "mailers" - ]; + jobClasses = [ "mailers" ]; threads = 5; }; }; @@ -121,12 +132,12 @@ in user = smtpCfg.email.address1; }; }; + caddy = { enable = true; virtualHosts = { ":80" = { extraConfig = '' - # Remove the outer http:// block wrapper handle_path /system/* { file_server * { root /var/lib/mastodon/public-system @@ -185,12 +196,18 @@ in }; }; - users.users.${serviceCfg.name}.extraGroups = [ - "postgres" - ]; - users.users.caddy.extraGroups = [ - serviceCfg.name - ]; + users.users = { + ${serviceCfg.name}.extraGroups = [ "postgres" ]; + caddy.extraGroups = [ serviceCfg.name ]; + fedifetcher = { + isSystemUser = true; + group = "fedifetcher"; + home = "/var/lib/fedifetcher"; + createHome = true; + }; + }; + + users.groups.fedifetcher = { }; networking.firewall.allowedTCPPorts = [ 22 # SSH @@ -204,21 +221,9 @@ in systemd = { services = { - mastodon-init-dirs = { - serviceConfig = { - PrivateMounts = lib.mkForce false; - }; - }; - mastodon-web = { - serviceConfig = { - PrivateMounts = lib.mkForce false; - }; - }; - mastodon-streaming-1 = { - serviceConfig = { - PrivateMounts = lib.mkForce false; - }; - }; + mastodon-init-dirs.serviceConfig.PrivateMounts = lib.mkForce false; + mastodon-web.serviceConfig.PrivateMounts = lib.mkForce false; + mastodon-streaming-1.serviceConfig.PrivateMounts = lib.mkForce false; mastodon-streaming-2.serviceConfig.PrivateMounts = lib.mkForce false; mastodon-streaming-3.serviceConfig.PrivateMounts = lib.mkForce false; mastodon-streaming-4.serviceConfig.PrivateMounts = lib.mkForce false; @@ -230,6 +235,7 @@ in mastodon-sidekiq-ingress.serviceConfig.PrivateMounts = lib.mkForce false; mastodon-sidekiq-mailers.serviceConfig.PrivateMounts = lib.mkForce false; mastodon-sidekiq-push-pull.serviceConfig.PrivateMounts = lib.mkForce false; + mastodon-copy-secrets = { description = "Copy secrets from virtiofs to local filesystem"; before = [ "mastodon-init-dirs.service" ]; @@ -246,11 +252,73 @@ in cp /run/secrets/database /etc/mastodon-secrets/database cp /run/secrets/redis /etc/mastodon-secrets/redis cp /run/secrets/smtp /etc/mastodon-secrets/smtp + cp /run/secrets/fedifetcher-token /etc/mastodon-secrets/fedifetcher-token chmod 755 /etc/mastodon-secrets chmod 644 /etc/mastodon-secrets/* ''; }; + + fedifetcher = { + description = "FediFetcher - Fetch missing posts for Mastodon"; + after = [ + "network-online.target" + "mastodon-web.service" + ]; + wants = [ "network-online.target" ]; + + serviceConfig = { + Type = "oneshot"; + User = "fedifetcher"; + Group = "fedifetcher"; + WorkingDirectory = "/var/lib/fedifetcher"; + TimeoutStartSec = "300"; + PrivateTmp = true; + NoNewPrivileges = true; + ProtectSystem = "strict"; + ProtectHome = true; + ReadWritePaths = "/var/lib/fedifetcher"; + ExecStart = + let + script = pkgs.writeShellScript "fedifetcher-run" '' + set -e + + # Wait for Mastodon to be fully ready + for i in {1..30}; do + if ${pkgs.curl}/bin/curl -sf http://localhost:80/health >/dev/null 2>&1; then + echo "Mastodon is ready" + break + fi + echo "Waiting for Mastodon to be ready... ($i/30)" + sleep 2 + done + + export ACCESS_TOKEN=$(cat /etc/mastodon-secrets/fedifetcher-token) + ${fedifetcherPython}/bin/python ${fedifetcherSrc}/find_posts.py \ + -c=${fedifetcherConfig} \ + --access-token="$ACCESS_TOKEN" + ''; + in + "${script}"; + }; + }; + + mastodon-init-db.serviceConfig.EnvironmentFile = "/var/lib/mastodon/.secrets_env"; + systemd-tmpfiles-setup.after = [ "var-lib-mastodon.mount" ]; }; + + timers.fedifetcher = { + description = "Timer for FediFetcher"; + wantedBy = [ "timers.target" ]; + + timerConfig = { + OnBootSec = "10min"; + OnUnitActiveSec = "15min"; + Unit = "fedifetcher.service"; + Persistent = true; + AccuracySec = "1min"; + }; + }; + network = { enable = true; networks."20-lan" = { @@ -268,14 +336,7 @@ in ]; }; }; - services = { - mastodon-init-db = { - serviceConfig = { - EnvironmentFile = "/var/lib/mastodon/.secrets_env"; - }; - }; - systemd-tmpfiles-setup.after = [ "var-lib-mastodon.mount" ]; - }; + tmpfiles.rules = [ "d /var/lib/mastodon 0755 mastodon mastodon -" "Z /var/lib/mastodon 0755 mastodon mastodon -" @@ -286,9 +347,12 @@ in "d /var/lib/mastodon/public-system/media_attachments 0755 mastodon mastodon -" "d /var/lib/mastodon/public-system/media_attachments/files 0755 mastodon mastodon -" "d /var/lib/mastodon/public-system/site_uploads 0755 mastodon mastodon -" + "d /var/lib/fedifetcher 0755 fedifetcher fedifetcher -" ]; }; + environment.systemPackages = [ fedifetcherPython ]; + microvm = { vcpu = 2; mem = 1024 * 3; @@ -359,6 +423,7 @@ in "database" "redis" "pass" + "fedifetcher-token" ] ); };