home-manager: optionally use putter for file management

The Putter tool is a Rust implementation of the file management
component of Home Manager. It takes a JSON manifest that indicates how
files should be symlinked and when the manifest is applied, does its
best to ensure that the file system reflect the manifest.

Putter offers some additional features that Home Manager does not
expose today, such as file copying (potentially recursive) and support
for overriding files within a recursive tree (e.g., inside a symlinked
tree one could ensure that a specific file is copies with specific
permissions).

This is considered highly experimental at the moment and to use Putter
one must set a hidden option.

modules/files.nix

@@ -55,6 +55,8 @@ let
         name = sourceName;
       };
 
+  putterStatePath = "${config.xdg.stateHome}/home-manager/putter-state.json";
+
 in
 
 {
@@ -88,11 +90,44 @@ in
       '';
     };
 
+    home.fileActivator = lib.mkOption {
+      type =
+        with lib.types;
+        enum [
+          "legacy"
+          "putter"
+        ];
+      default = "legacy";
+      example = "putter";
+      visible = false;
+      description = ''
+        The tooling to use to place files during activation.
+
+        The legacy option (currently the default) is the built-in tooling that
+        is very robust, but is limited in future potential.
+
+        The putter option is a new external tool that may replace the legacy
+        alternative in the future. It is not as hardened as the legacy
+        alternative but will allow future features such as file copying.
+
+        This option should be considered experimental and is therefore hidden
+        from documentation at this time.
+      '';
+    };
+
     home-files = lib.mkOption {
       type = lib.types.package;
       internal = true;
       description = "Package to contain all home files";
     };
+
+    home.internal = {
+      filePutterConfig = lib.mkOption {
+        type = lib.types.package;
+        internal = true;
+        description = "Putter configuration.";
+      };
+    };
   };
 
   config = {
@@ -149,21 +184,29 @@ in
 
         storeDir = lib.escapeShellArg builtins.storeDir;
 
-        check = pkgs.replaceVars ./files/check-link-targets.sh {
+        legacyCheckScript = pkgs.replaceVars ./files/check-link-targets.sh {
           inherit (config.lib.bash) initHomeManagerLib;
           inherit forcedPaths storeDir;
         };
-      in
-      ''
-        function checkNewGenCollision() {
-          local newGenFiles
-          newGenFiles="$(readlink -e "$newGenPath/home-files")"
-          find "$newGenFiles" \( -type f -or -type l \) \
-              -exec bash ${check} "$newGenFiles" {} +
-        }
 
-        checkNewGenCollision || exit 1
-      ''
+        legacyCheckLinkTargets = ''
+          function checkNewGenCollision() {
+            local newGenFiles
+            newGenFiles="$(readlink -e "$newGenPath/home-files")"
+            find "$newGenFiles" \( -type f -or -type l \) \
+                -exec bash ${legacyCheckScript} "$newGenFiles" {} +
+          }
+
+          checkNewGenCollision || exit 1
+        '';
+
+        putterCheckLinkTargets = ''
+          ${lib.getExe pkgs.putter} check $VERBOSE_ARG \
+            --state-file "${putterStatePath}" \
+            ${config.home.internal.filePutterConfig}
+        '';
+      in
+      if config.home.fileActivator == "putter" then putterCheckLinkTargets else legacyCheckLinkTargets
     );
 
     # This activation script will
@@ -186,7 +229,9 @@ in
     # source and target generation.
     home.activation.linkGeneration = lib.hm.dag.entryAfter [ "writeBoundary" ] (
       let
-        link = pkgs.writeShellScript "link" ''
+        storeDir = lib.escapeShellArg builtins.storeDir;
+
+        legacyLink = pkgs.writeShellScript "link" ''
           ${config.lib.bash.initHomeManagerLib}
 
           newGenFiles="$1"
@@ -220,12 +265,12 @@ in
           done
         '';
 
-        cleanup = pkgs.writeShellScript "cleanup" ''
+        legacyCleanup = pkgs.writeShellScript "cleanup" ''
           ${config.lib.bash.initHomeManagerLib}
 
           # A symbolic link whose target path matches this pattern will be
           # considered part of a Home Manager generation.
-          homeFilePattern="$(readlink -e ${lib.escapeShellArg builtins.storeDir})/*-home-manager-files/*"
+          homeFilePattern="$(readlink -e ${storeDir})/*-home-manager-files/*"
 
           newGenFiles="$1"
           shift 1
@@ -256,38 +301,86 @@ in
             fi
           done
         '';
+
+        # If Putter is not enabled, then generate a fake state file to allow
+        # switching to Putter in the future.
+        putterCompatState =
+          let
+            putter = import ./lib/putter.nix { inherit lib; };
+            manifest = putter.mkPutterCompatState {
+              sourceBaseDirectory = config.home-files;
+              targetBaseDirectory = config.home.homeDirectory;
+              fileEntries = cfg;
+            };
+          in
+          pkgs.writeText "hm-putter-state.json" manifest;
+
+        # This activation script will
+        #
+        # 1. Remove files from the old generation that are not in the new
+        #    generation.
+        #
+        # 2. Symlink files from the new generation into $HOME.
+        #
+        # This order is needed to ensure that we always know which links
+        # belong to which generation. Specifically, if we're moving from
+        # generation A to generation B having sets of home file links FA
+        # and FB, respectively then cleaning before linking produces state
+        # transitions similar to
+        #
+        #      FA   →   FA ∩ FB   →   (FA ∩ FB) ∪ FB = FB
+        #
+        # and a failure during the intermediate state FA ∩ FB will not
+        # result in lost links because this set of links are in both the
+        # source and target generation.
+        legacyLinkGeneration = ''
+          function linkNewGen() {
+            _i "Creating home file links in %s" "$HOME"
+
+            local newGenFiles
+            newGenFiles="$(readlink -e "$newGenPath/home-files")"
+            find "$newGenFiles" \( -type f -or -type l \) \
+              -exec bash ${legacyLink} "$newGenFiles" {} +
+
+            # Copy in the Putter compatible state file. This is to allow a later
+            # switchover to Putter.
+            run install -Dp -m600 $VERBOSE_ARG ${
+              lib.escapeShellArgs [
+                putterCompatState
+                putterStatePath
+              ]
+            }
+          }
+
+          function cleanOldGen() {
+            if [[ ! -v oldGenPath || ! -e "$oldGenPath/home-files" ]] ; then
+              return
+            fi
+
+            _i "Cleaning up orphan links from %s" "$HOME"
+
+            local newGenFiles oldGenFiles
+            newGenFiles="$(readlink -e "$newGenPath/home-files")"
+            oldGenFiles="$(readlink -e "$oldGenPath/home-files")"
+
+            # Apply the cleanup script on each leaf in the old
+            # generation. The find command below will print the
+            # relative path of the entry.
+            find "$oldGenFiles" '(' -type f -or -type l ')' -printf '%P\0' \
+              | xargs -0 bash ${legacyCleanup} "$newGenFiles"
+          }
+
+          cleanOldGen
+          linkNewGen
+        '';
+
+        putterLinkGeneration = ''
+          ${lib.getExe pkgs.putter} apply $VERBOSE_ARG ''${DRY_RUN:+--dry-run} \
+            --state-file "${putterStatePath}" \
+            ${config.home.internal.filePutterConfig}
+        '';
       in
-      ''
-        function linkNewGen() {
-          _i "Creating home file links in %s" "$HOME"
-
-          local newGenFiles
-          newGenFiles="$(readlink -e "$newGenPath/home-files")"
-          find "$newGenFiles" \( -type f -or -type l \) \
-            -exec bash ${link} "$newGenFiles" {} +
-        }
-
-        function cleanOldGen() {
-          if [[ ! -v oldGenPath || ! -e "$oldGenPath/home-files" ]] ; then
-            return
-          fi
-
-          _i "Cleaning up orphan links from %s" "$HOME"
-
-          local newGenFiles oldGenFiles
-          newGenFiles="$(readlink -e "$newGenPath/home-files")"
-          oldGenFiles="$(readlink -e "$oldGenPath/home-files")"
-
-          # Apply the cleanup script on each leaf in the old
-          # generation. The find command below will print the
-          # relative path of the entry.
-          find "$oldGenFiles" '(' -type f -or -type l ')' -printf '%P\0' \
-            | xargs -0 bash ${cleanup} "$newGenFiles"
-        }
-
-        cleanOldGen
-        linkNewGen
-      ''
+      if config.home.fileActivator == "putter" then putterLinkGeneration else legacyLinkGeneration
     );
 
     home.activation.checkFilesChanged = lib.hm.dag.entryBefore [ "linkGeneration" ] (
@@ -334,6 +427,18 @@ in
       '') (lib.filter (v: v.onChange != "") cfg)
     );
 
+    home.internal.filePutterConfig =
+      let
+        putter = import ./lib/putter.nix { inherit lib; };
+        manifest = putter.mkPutterManifest {
+          inherit putterStatePath;
+          sourceBaseDirectory = config.home-files;
+          targetBaseDirectory = config.home.homeDirectory;
+          fileEntries = cfg;
+        };
+      in
+      pkgs.writeText "hm-putter.json" manifest;
+
     # Symlink directories and files that have the right execute bit.
     # Copy files that need their execute bit changed.
     home-files =

modules/home-environment.nix

@@ -917,6 +917,7 @@ in
             --subst-var-by GENERATION_DIR $out
 
           ln -s ${config.home-files} $out/home-files
+          ln -s ${config.home.internal.filePutterConfig} $out/putter.json
           ln -s ${cfg.path} $out/home-path
 
           cp "$extraDependenciesPath" "$out/extra-dependencies"

modules/lib/putter.nix

@@ -0,0 +1,82 @@
+# Contains some handy functions for generating Putter file manifests.
+
+{ lib }:
+
+let
+
+  inherit (lib)
+    filter
+    hasPrefix
+    optionalAttrs
+    ;
+
+in
+{
+  # Converts a Home Manager style list of file specifications into a Putter
+  # configuration.
+  #
+  # Note, the interface of this function is not considered stable, it may change
+  # as the needs of Home Manager change.
+  mkPutterManifest =
+    {
+      putterStatePath,
+      sourceBaseDirectory,
+      targetBaseDirectory,
+      fileEntries,
+    }:
+    let
+      # Create a Putter entry for the given file.
+      mkEntry =
+        f:
+        {
+          source = "${sourceBaseDirectory}/${f.target}";
+          # source = "${f.source}";
+          target = (if hasPrefix "/" f.target then "" else "${targetBaseDirectory}/") + f.target;
+        }
+        // optionalAttrs f.force {
+          collision.resolution = "force";
+        }
+        // optionalAttrs f.recursive {
+          action.type = "recursive_symlink";
+        };
+
+      putterJson = {
+        version = "1";
+        state = putterStatePath;
+        files = map mkEntry (filter (f: f.enable) fileEntries);
+      };
+
+      putterJsonText = builtins.toJSON putterJson;
+    in
+    putterJsonText;
+
+  # Create a putter state file to allow compatibility between legacy and putter
+  # managed files.
+  #
+  # Note, the interface of this function is not considered stable, it may change
+  # as the needs of Home Manager change.
+  mkPutterCompatState =
+    {
+      sourceBaseDirectory,
+      targetBaseDirectory,
+      fileEntries,
+    }:
+    let
+      mkEntry = f: {
+        name = (if hasPrefix "/" f.target then "" else "${targetBaseDirectory}/") + f.target;
+        value = {
+          source = "${sourceBaseDirectory}/${f.target}";
+          modified = {
+            secs_since_epoch = 0;
+            nanos_since_epoch = 0;
+          };
+        };
+      };
+
+      stateJsonText = builtins.toJSON {
+        version = "1";
+        files = lib.listToAttrs (map mkEntry (filter (f: f.enable) fileEntries));
+      };
+    in
+    stateJsonText;
+}

tests/default.nix

@@ -148,6 +148,8 @@ let
 
           test.enableBig = enableBig;
           test.enableLegacyIfd = enableLegacyIfd;
+
+          home.fileActivator = "putter";
         }
       )
     ];

tests/integration/default.nix

@@ -20,6 +20,7 @@ let
     nh = runTest ./standalone/nh.nix;
     nixos-basics = runTest ./nixos/basics.nix;
     nixos-legacy-profile-management = runTest ./nixos/legacy-profile-management.nix;
+    putter = runTest ./standalone/putter.nix;
     rclone = runTest ./standalone/rclone;
     rclone-sops-nix = runTest ./standalone/rclone/sops-nix.nix;
     rclone-agenix = runTest ./standalone/rclone/agenix.nix;

tests/integration/standalone/alice-home-file-activator.nix

@@ -0,0 +1,10 @@
+{
+  home.username = "alice";
+  home.homeDirectory = "/home/alice";
+  home.stateVersion = "24.11";
+  home.file.test.text = "test";
+  programs.home-manager.enable = true;
+
+  # Enable specific file activator.
+  home.fileActivator = "@fileActivator@";
+}

tests/integration/standalone/putter.nix

@@ -0,0 +1,94 @@
+{ pkgs, ... }:
+
+{
+  name = "standalone-putter";
+  meta.maintainers = [ pkgs.lib.maintainers.rycee ];
+
+  nodes.machine = {
+    imports = [ "${pkgs.path}/nixos/modules/installer/cd-dvd/channel.nix" ];
+    virtualisation.memorySize = 2048;
+    users.users.alice = {
+      isNormalUser = true;
+      description = "Alice Foobar";
+      password = "foobar";
+      uid = 1000;
+    };
+  };
+
+  testScript = ''
+    start_all()
+    machine.wait_for_unit("network.target")
+    machine.wait_for_unit("multi-user.target")
+
+    home_manager = "${../../..}"
+
+    def login_as_alice():
+      machine.wait_until_tty_matches("1", "login: ")
+      machine.send_chars("alice\n")
+      machine.wait_until_tty_matches("1", "Password: ")
+      machine.send_chars("foobar\n")
+      machine.wait_until_tty_matches("1", "alice\\@machine")
+
+    def logout_alice():
+      machine.send_chars("exit\n")
+
+    def alice_cmd(cmd):
+      return f"su -l alice --shell /bin/sh -c $'export XDG_RUNTIME_DIR=/run/user/$UID ; {cmd}'"
+
+    def succeed_as_alice(cmd):
+      return machine.succeed(alice_cmd(cmd))
+
+    def fail_as_alice(cmd):
+      return machine.fail(alice_cmd(cmd))
+
+    # Create a persistent login so that Alice has a systemd session.
+    login_as_alice()
+
+    # Set up a home-manager channel.
+    succeed_as_alice(" ; ".join([
+      "mkdir -p /home/alice/.nix-defexpr/channels",
+      f"ln -s {home_manager} /home/alice/.nix-defexpr/channels/home-manager"
+    ]))
+
+    with subtest("Home Manager installation"):
+      succeed_as_alice("nix-shell \"<home-manager>\" -A install")
+
+      actual = machine.succeed("ls /home/alice/.config/home-manager")
+      assert actual == "home.nix\n", \
+        f"unexpected content of /home/alice/.config/home-manager: {actual}"
+
+      machine.succeed("diff -u ${./alice-home-init.nix} /home/alice/.config/home-manager/home.nix")
+
+      # The default configuration creates this link on activation.
+      machine.succeed("test -L /home/alice/.cache/.keep")
+
+    with subtest("Activate with Putter"):
+      succeed_as_alice("cp ${
+        pkgs.substitute {
+          src = ./alice-home-file-activator.nix;
+          substitutions = [
+            "--replace"
+            "@fileActivator@"
+            "putter"
+          ];
+        }
+      } /home/alice/.config/home-manager/home.nix")
+
+      succeed_as_alice("home-manager switch")
+
+      machine.succeed("test -L /home/alice/test")
+
+    with subtest("Home Manager uninstallation"):
+      succeed_as_alice("yes | home-manager uninstall -L")
+
+      machine.succeed("test ! -e /home/alice/.cache/.keep")
+      machine.succeed("test ! -e /home/alice/.cache/test")
+
+      # TODO: Fix uninstall to fully remove the share directory.
+      machine.succeed("test ! -e /home/alice/.local/share/home-manager/gcroots")
+      machine.succeed("test ! -e /home/alice/.local/state/home-manager")
+      machine.succeed("test ! -e /home/alice/.local/state/nix/profiles/home-manager")
+
+    logout_alice()
+  '';
+}