feat: added fedifetcher

modules/nixos/guests/defenseio/default.nix

@@ -75,12 +75,12 @@ in
               ];
               mem =
                 let
-                  num = 1024;
+                  num = num: (num * 1024);
-                  ceresRam = num * 50;
+                  ceresRam = num 50;
-                  erisRam = num * 7;
+                  erisRam = num 7;
-                  marsRam = num * 22;
+                  marsRam = num 22;
-                  deimosRam = num * 7;
+                  deimosRam = num 7;
-                  phobosRam = num * 7;
+                  phobosRam = num 7;
                 in
                 deviceLogic ceresRam erisRam marsRam deimosRam phobosRam;
               shares = [

modules/nixos/guests/mastodon/default.nix

@@ -14,32 +14,49 @@ let
   host = serviceCfg.domains.url0;
   dns0 = instances.web.dns.provider0;
   dns0Path = "dns/${dns0}";
+
+  fedifetcherConfig = pkgs.writeText "fedifetcher-config.json" (
+    builtins.toJSON {
+      server = "https://${host}";
+      home-timeline-length = 200;
+      max-followings = 80;
+      from-notifications = 1;
+      max-bookmarks = 80;
+      max-favourites = 40;
+      backfill-with-context = 1;
+      backfill-mentioned-users = 1;
+      remember-users-for-hours = 168;
+      remember-hosts-for-days = 30;
+      http-timeout = 5;
+      lock-hours = 24;
+      log-level = "INFO";
+    }
+  );
+
+  fedifetcherPython = pkgs.python3.withPackages (
+    ps: with ps; [
+      requests
+      pytz
+      beautifulsoup4
+    ]
+  );
+
+  fedifetcherSrc = pkgs.fetchFromGitHub {
+    owner = "nanos";
+    repo = "FediFetcher";
+    rev = "main";
+    hash = "sha256-/J7psV/mA7okuuO7/aXVVWS9p63eMncG2CEEGN38ip0=";
+  };
 in
 {
   # If you need to start fresh for some reason, run these to create the new Admin account:
   # sudo -u mastodon mastodon-tootctl accounts create nick --email=nick@localhost --confirmed --role=Owner
   # sudo -u mastodon mastodon-tootctl accounts approve nick
-
   # If you fuck up and lose the password, use this:
   # sudo mastodon-tootctl accounts modify --reset-password nick
-
   # If you really fuck up and name yourself wrong, use this shit
   # sudo mastodon-tootctl accounts modify username --remove-role
 
-  # nixpkgs.overlays = [
-  #   (
-  #     final: prev: {
-  #       mastodon = prev.mastodon.overrideAttrs (oldAttrs: {
-  #         postPatch =
-  #           (oldAttrs.postPatch or "")
-  #           + ''
-  #             patch -p1 < ${./chars.patch}
-  #           '';
-  #       });
-  #     }
-  #   )
-  # ];
-
   microvm.vms = {
     ${serviceCfg.name} = {
       autostart = true;
@@ -48,6 +65,7 @@ in
         system.stateVersion = "24.05";
         time.timeZone = "America/Winnipeg";
         users.users.root.openssh.authorizedKeys.keys = flake.config.people.users.${user0}.sshKeys;
+
         services = {
           ${serviceCfg.name} = {
             enable = true;
@@ -81,20 +99,15 @@ in
             sidekiqThreads = 25;
             sidekiqProcesses = {
               all = {
-                jobClasses = [
+                jobClasses = [ ];
-                ];
                 threads = null;
               };
               default = {
-                jobClasses = [
+                jobClasses = [ "default" ];
-                  "default"
-                ];
                 threads = 5;
               };
               ingress = {
-                jobClasses = [
+                jobClasses = [ "ingress" ];
-                  "ingress"
-                ];
                 threads = 5;
               };
               push-pull = {
@@ -105,9 +118,7 @@ in
                 threads = 5;
               };
               mailers = {
-                jobClasses = [
+                jobClasses = [ "mailers" ];
-                  "mailers"
-                ];
                 threads = 5;
               };
             };
@@ -121,12 +132,12 @@ in
               user = smtpCfg.email.address1;
             };
           };
+
           caddy = {
             enable = true;
             virtualHosts = {
               ":80" = {
                 extraConfig = ''
-                  # Remove the outer http:// block wrapper
                   handle_path /system/* {
                     file_server * {
                       root /var/lib/mastodon/public-system
@@ -185,12 +196,18 @@ in
           };
         };
 
-        users.users.${serviceCfg.name}.extraGroups = [
+        users.users = {
-          "postgres"
+          ${serviceCfg.name}.extraGroups = [ "postgres" ];
-        ];
+          caddy.extraGroups = [ serviceCfg.name ];
-        users.users.caddy.extraGroups = [
+          fedifetcher = {
-          serviceCfg.name
+            isSystemUser = true;
-        ];
+            group = "fedifetcher";
+            home = "/var/lib/fedifetcher";
+            createHome = true;
+          };
+        };
+
+        users.groups.fedifetcher = { };
 
         networking.firewall.allowedTCPPorts = [
           22 # SSH
@@ -204,21 +221,9 @@ in
 
         systemd = {
           services = {
-            mastodon-init-dirs = {
+            mastodon-init-dirs.serviceConfig.PrivateMounts = lib.mkForce false;
-              serviceConfig = {
+            mastodon-web.serviceConfig.PrivateMounts = lib.mkForce false;
-                PrivateMounts = lib.mkForce false;
+            mastodon-streaming-1.serviceConfig.PrivateMounts = lib.mkForce false;
-              };
-            };
-            mastodon-web = {
-              serviceConfig = {
-                PrivateMounts = lib.mkForce false;
-              };
-            };
-            mastodon-streaming-1 = {
-              serviceConfig = {
-                PrivateMounts = lib.mkForce false;
-              };
-            };
             mastodon-streaming-2.serviceConfig.PrivateMounts = lib.mkForce false;
             mastodon-streaming-3.serviceConfig.PrivateMounts = lib.mkForce false;
             mastodon-streaming-4.serviceConfig.PrivateMounts = lib.mkForce false;
@@ -230,6 +235,7 @@ in
             mastodon-sidekiq-ingress.serviceConfig.PrivateMounts = lib.mkForce false;
             mastodon-sidekiq-mailers.serviceConfig.PrivateMounts = lib.mkForce false;
             mastodon-sidekiq-push-pull.serviceConfig.PrivateMounts = lib.mkForce false;
+
             mastodon-copy-secrets = {
               description = "Copy secrets from virtiofs to local filesystem";
               before = [ "mastodon-init-dirs.service" ];
@@ -246,11 +252,73 @@ in
                 cp /run/secrets/database /etc/mastodon-secrets/database
                 cp /run/secrets/redis /etc/mastodon-secrets/redis
                 cp /run/secrets/smtp /etc/mastodon-secrets/smtp
+                cp /run/secrets/fedifetcher-token /etc/mastodon-secrets/fedifetcher-token
                 chmod 755 /etc/mastodon-secrets
                 chmod 644 /etc/mastodon-secrets/*
               '';
             };
+
+            fedifetcher = {
+              description = "FediFetcher - Fetch missing posts for Mastodon";
+              after = [
+                "network-online.target"
+                "mastodon-web.service"
+              ];
+              wants = [ "network-online.target" ];
+
+              serviceConfig = {
+                Type = "oneshot";
+                User = "fedifetcher";
+                Group = "fedifetcher";
+                WorkingDirectory = "/var/lib/fedifetcher";
+                TimeoutStartSec = "300";
+                PrivateTmp = true;
+                NoNewPrivileges = true;
+                ProtectSystem = "strict";
+                ProtectHome = true;
+                ReadWritePaths = "/var/lib/fedifetcher";
+                ExecStart =
+                  let
+                    script = pkgs.writeShellScript "fedifetcher-run" ''
+                      set -e
+
+                      # Wait for Mastodon to be fully ready
+                      for i in {1..30}; do
+                        if ${pkgs.curl}/bin/curl -sf http://localhost:80/health >/dev/null 2>&1; then
+                          echo "Mastodon is ready"
+                          break
+                        fi
+                        echo "Waiting for Mastodon to be ready... ($i/30)"
+                        sleep 2
+                      done
+
+                      export ACCESS_TOKEN=$(cat /etc/mastodon-secrets/fedifetcher-token)
+                      ${fedifetcherPython}/bin/python ${fedifetcherSrc}/find_posts.py \
+                        -c=${fedifetcherConfig} \
+                        --access-token="$ACCESS_TOKEN"
+                    '';
+                  in
+                  "${script}";
+              };
+            };
+
+            mastodon-init-db.serviceConfig.EnvironmentFile = "/var/lib/mastodon/.secrets_env";
+            systemd-tmpfiles-setup.after = [ "var-lib-mastodon.mount" ];
           };
+
+          timers.fedifetcher = {
+            description = "Timer for FediFetcher";
+            wantedBy = [ "timers.target" ];
+
+            timerConfig = {
+              OnBootSec = "10min";
+              OnUnitActiveSec = "15min";
+              Unit = "fedifetcher.service";
+              Persistent = true;
+              AccuracySec = "1min";
+            };
+          };
+
           network = {
             enable = true;
             networks."20-lan" = {
@@ -268,14 +336,7 @@ in
               ];
             };
           };
-          services = {
+
-            mastodon-init-db = {
-              serviceConfig = {
-                EnvironmentFile = "/var/lib/mastodon/.secrets_env";
-              };
-            };
-            systemd-tmpfiles-setup.after = [ "var-lib-mastodon.mount" ];
-          };
           tmpfiles.rules = [
             "d /var/lib/mastodon 0755 mastodon mastodon -"
             "Z /var/lib/mastodon 0755 mastodon mastodon -"
@@ -286,9 +347,12 @@ in
             "d /var/lib/mastodon/public-system/media_attachments 0755 mastodon mastodon -"
             "d /var/lib/mastodon/public-system/media_attachments/files 0755 mastodon mastodon -"
             "d /var/lib/mastodon/public-system/site_uploads 0755 mastodon mastodon -"
+            "d /var/lib/fedifetcher 0755 fedifetcher fedifetcher -"
           ];
         };
 
+        environment.systemPackages = [ fedifetcherPython ];
+
         microvm = {
           vcpu = 2;
           mem = 1024 * 3;
@@ -359,6 +423,7 @@ in
           "database"
           "redis"
           "pass"
+          "fedifetcher-token"
         ]
     );
   };