feat(action-plugin): restore dynamic actions from the shared cache

otherlibs/dune-rpc-lwt/test/action-plugin/one-dependency/bin/foo.ml

@@ -145,6 +145,14 @@ let action dap =
          | _ -> failwith "helper failed")
   | [| _ |] -> ordinary_action dap ~path:"some_dependency"
   | [| _; "read"; path |] -> ordinary_action dap ~path
+  | [| _; "choose" |] ->
+    let open Lwt.Syntax in
+    let* path = read_file dap ~path:"choice" in
+    ordinary_action dap ~path:(String.trim path)
+  | [| _; "glob"; path; pattern |] ->
+    let open Lwt.Syntax in
+    let* files = read_directory_with_glob dap ~path ~glob:(Glob.of_string pattern) in
+    Lwt_list.iter_s Lwt_io.printl files
   | [| _; "sandbox" |] -> sandbox_action dap
   | [| _; "detached"; state |] -> detached_action dap state
   | [| _; "hold"; connection; release |] -> held_action dap ~connection ~release

otherlibs/dune-rpc-lwt/test/action-plugin/one-dependency/cache.t

@@ -1,7 +1,9 @@
-Dynamic actions currently rerun in a fresh workspace even with a shared cache.
+A fresh workspace replays dependency discovery before restoring dynamic outputs.
+The cache trace reports discovery misses as well as artifact misses.
 
   $ export DUNE_CACHE_ROOT="$PWD/.cache"
   $ export DUNE_CACHE=enabled
+  $ export DUNE_TRACE=+cache
   $ mkdir template
   $ cp bin/foo.exe template/
   $ cat > template/dune-project <<'EOF'
@@ -21,11 +23,176 @@ Dynamic actions currently rerun in a fresh workspace even with a shared cache.
   >     | select(.args.prog | endswith("foo.exe"))
   >     | .args | {prog: (.prog | split("/") | last), process_args, exit}'
   > }
+  $ show_misses () {
+  >   dune trace cat | jq -c '
+  >     select(.cat == "cache" and .name == "miss")
+  >     | select(.args.head | endswith("/output"))
+  >     | {name, reason: .args.reason}'
+  > }
   $ cp -R template first
   $ (cd first && dune build --root . output && show_runs)
   {"prog":"foo.exe","process_args":["read","input"],"exit":0}
+  $ (cd first && show_misses)
+  {"name":"miss","reason":"dynamic dependency manifest unavailable"}
   $ cp -R template second
   $ (cd second && dune build --root . output && show_runs)
-  {"prog":"foo.exe","process_args":["read","input"],"exit":0}
+  $ (cd second && show_misses)
   $ cat second/_build/default/output
   first
+
+Changed inputs select a new branch; old branches remain reusable.
+
+  $ printf changed > template/source
+  $ cp -R template changed
+  $ (cd changed && dune build --root . output && show_runs)
+  {"prog":"foo.exe","process_args":["read","input"],"exit":0}
+  $ cat changed/_build/default/output
+  changed
+  $ (cd changed && show_misses)
+  {"name":"miss","reason":"dynamic dependency manifest unavailable"}
+  $ cp -R template changed-again
+  $ (cd changed-again && dune build --root . output && show_runs)
+  $ printf first > template/source
+  $ cp -R template old-branch
+  $ (cd old-branch && dune build --root . output && show_runs)
+
+A shared hit also populates the workspace cache with the dynamic dependencies.
+
+  $ (cd second && dune build --root . output && show_runs)
+  $ printf local-change > second/source
+  $ (cd second && dune build --root . output && show_runs)
+  {"prog":"foo.exe","process_args":["read","input"],"exit":0}
+  $ cat second/_build/default/output
+  local-change
+
+Copy-mode storage works too.
+
+  $ export DUNE_CACHE_STORAGE_MODE=copy
+  $ printf copy-mode > template/source
+  $ cp -R template copy-first
+  $ (cd copy-first && dune build --root . output && show_runs)
+  {"prog":"foo.exe","process_args":["read","input"],"exit":0}
+  $ cp -R template copy-again
+  $ (cd copy-again && dune build --root . output && show_runs)
+  $ cat copy-again/_build/default/output
+  copy-mode
+
+Artifact lookup already reports misses for reproducibility checks.
+
+  $ cp -R template check
+  $ (cd check && dune build --root . output --cache-check-probability=1.0 && show_misses)
+  {"name":"miss","reason":"rerunning for reproducibility check"}
+
+An earlier request can select a different dependency. Replay must stop before
+trying to build the now-absent input from the old branch.
+
+  $ cat > template/dune <<'EOF'
+  > (rule
+  >  (target output)
+  >  (action (with-stdout-to output (dynamic-run ./foo.exe choose))))
+  > EOF
+  $ printf left > template/choice
+  $ printf left-value > template/left
+  $ cp -R template choose-left
+  $ (cd choose-left && dune build --root . output && show_runs)
+  {"prog":"foo.exe","process_args":["choose"],"exit":0}
+  $ printf right > template/choice
+  $ rm template/left
+  $ printf right-value > template/right
+  $ cp -R template choose-right
+  $ (cd choose-right && dune build --root . output && show_runs)
+  {"prog":"foo.exe","process_args":["choose"],"exit":0}
+  $ cat choose-right/_build/default/output
+  right-value
+  $ cp -R template choose-again
+  $ (cd choose-again && dune build --root . output && show_runs)
+
+Glob requests are replayed, including changes to matching files and names.
+
+  $ cat > template/dune <<'EOF'
+  > (rule (target listed-generated) (action (write-file listed-generated generated)))
+  > (rule
+  >  (target output)
+  >  (action (with-stdout-to output (dynamic-run ./foo.exe glob . listed*))))
+  > EOF
+  $ printf one > template/listed-source
+  $ cp -R template glob-first
+  $ (cd glob-first && dune build --root . output && show_runs)
+  {"prog":"foo.exe","process_args":["glob",".","listed*"],"exit":0}
+  $ cp -R template glob-again
+  $ (cd glob-again && dune build --root . output && show_runs)
+  $ cat glob-again/_build/default/output
+  listed-generated
+  listed-source
+  $ printf two > template/listed-source
+  $ cp -R template glob-changed
+  $ (cd glob-changed && dune build --root . output && show_runs)
+  {"prog":"foo.exe","process_args":["glob",".","listed*"],"exit":0}
+  $ printf extra > template/listed-added
+  $ cp -R template glob-added
+  $ (cd glob-added && dune build --root . output && show_runs)
+  {"prog":"foo.exe","process_args":["glob",".","listed*"],"exit":0}
+  $ cat glob-added/_build/default/output
+  listed-added
+  listed-generated
+  listed-source
+
+A syntactically valid payload with an invalid manifest shape is also a miss.
+
+  $ grep -rl '^[(]4:deps' "$DUNE_CACHE_ROOT/db/files" | while read entry; do
+  >   chmod u+w "$entry"
+  >   printf '(7:unknown())' > "$entry"
+  > done
+  $ cp -R template invalid-manifest
+  $ (cd invalid-manifest && dune build --root . output && show_runs)
+  {"prog":"foo.exe","process_args":["glob",".","listed*"],"exit":0}
+  $ (cd invalid-manifest && show_misses)
+  {"name":"miss","reason":"dynamic dependency manifest unavailable"}
+  $ cat invalid-manifest/_build/default/output
+  listed-added
+  listed-generated
+  listed-source
+
+Ordinary trimming reclaims both manifest payloads and their metadata.
+
+  $ grep -rl '.dap-manifest' "$DUNE_CACHE_ROOT/db/meta" > /dev/null
+  $ dune cache trim --size 0B > /dev/null
+  $ grep -rl '.dap-manifest' "$DUNE_CACHE_ROOT/db/meta"
+  [1]
+  $ cp -R template trimmed
+  $ (cd trimmed && dune build --root . output && show_runs)
+  {"prog":"foo.exe","process_args":["glob",".","listed*"],"exit":0}
+  $ (cd trimmed && show_misses)
+  {"name":"miss","reason":"dynamic dependency manifest unavailable"}
+
+Corrupt discovery metadata is a miss, even when its artifacts remain cached.
+
+  $ grep -rl '.dap-manifest' "$DUNE_CACHE_ROOT/db/meta" | while read entry; do
+  >   chmod u+w "$entry"
+  >   printf broken > "$entry"
+  > done
+  $ cp -R template corrupt
+  $ (cd corrupt && dune build --root . output && show_runs)
+  {"prog":"foo.exe","process_args":["glob",".","listed*"],"exit":0}
+  $ cat corrupt/_build/default/output
+  listed-added
+  listed-generated
+  listed-source
+  $ (cd corrupt && show_misses)
+  {"name":"miss","reason":"dynamic dependency manifest unavailable"}
+
+Disabled caching and sandbox exclusion also report why no shared hit is possible.
+
+  $ cp -R template disabled
+  $ (cd disabled && DUNE_CACHE=disabled dune build --root . output && show_misses)
+  {"name":"miss","reason":"can't go in shared cache"}
+  $ cat > template/dune <<'EOF'
+  > (rule (target listed-generated) (action (write-file listed-generated generated)))
+  > (rule
+  >  (target output)
+  >  (deps (sandbox always))
+  >  (action (with-stdout-to output (dynamic-run ./foo.exe glob . listed*))))
+  > EOF
+  $ cp -R template sandboxed
+  $ (cd sandboxed && dune build --root . output --sandbox=hardlink && show_misses)
+  {"name":"miss","reason":"can't go in shared cache"}

src/dune_cache/local.ml

@@ -57,15 +57,20 @@ let restore_file_content path : string Restore_result.t =
     Error e
 ;;
 
-let restore_metadata_file file ~of_sexp : _ Restore_result.t =
+let restore_sexp file =
   restore_file_content file
   |> Restore_result.bind ~f:(fun content ->
     match Csexp.parse_string content with
     | Error (_offset, msg) -> Error (Failure msg)
-    | Ok sexp ->
-      (match of_sexp sexp with
-       | Ok content -> Restored content
-       | Error e -> Error e))
+    | Ok sexp -> Restored sexp)
+;;
+
+let restore_metadata_file file ~of_sexp : _ Restore_result.t =
+  restore_sexp file
+  |> Restore_result.bind ~f:(fun sexp ->
+    match of_sexp sexp with
+    | Ok content -> Restored content
+    | Error e -> Error e)
 ;;
 
 module Artifacts = struct

src/dune_cache/local.mli

@@ -41,6 +41,8 @@ module Restore_result : sig
   val bind : 'a t -> f:('a -> 'b t) -> 'b t
 end
 
+val restore_sexp : Path.t -> Sexp.t Restore_result.t
+
 (** An [Artifacts] entry corresponds to the targets produced by an action. *)
 module Artifacts : sig
   module Metadata_entry : sig

src/dune_cache/shared.ml

@@ -9,6 +9,54 @@ let config = ref Config.Disabled
 
 open Import
 
+module Dynamic_deps = struct
+  let manifest_path = Path.Local.of_string ".dap-manifest"
+
+  let load ~rule_digest =
+    match !config with
+    | Disabled -> None
+    | Enabled _ ->
+      (match Artifacts.list ~rule_digest with
+       | Restored [ { Artifacts.Metadata_entry.path; digest = Some digest } ]
+         when Path.Local.equal path manifest_path ->
+         (match
+            Local.restore_sexp (Lazy.force (Layout.file_path ~file_digest:digest))
+          with
+          | Restored value -> Some value
+          | Not_found_in_cache | Error _ -> None)
+       | Restored _ | Not_found_in_cache | Error _ -> None)
+  ;;
+
+  let store ~rule_digest manifest =
+    match !config with
+    | Disabled -> ()
+    | Enabled { storage_mode = mode; _ } ->
+      let content = Csexp.to_string manifest in
+      let digest =
+        Digest.path_with_executable_bit
+          ~executable:false
+          ~content_digest:(Digest.string content)
+      in
+      (match
+         match
+           let dst = Lazy.force (Layout.file_path ~file_digest:digest) in
+           Util.write_atomically ~mode ~content dst
+         with
+         | Error exn -> Store_result.Error exn
+         | Ok | Already_present ->
+           Artifacts.Metadata_file.store
+             [ { Artifacts.Metadata_entry.path = manifest_path; digest = Some digest } ]
+             ~mode
+             ~rule_digest
+       with
+       | Stored | Already_present | Will_not_store_due_to_non_determinism _ -> ()
+       | Error exn ->
+         Log.info
+           "Unable to store dynamic dependency manifest"
+           [ "error", Dyn.string (Printexc.to_string exn) ])
+  ;;
+end
+
 module Store_artifacts_result = struct
   type t =
     | Stored of Digest.t Targets.Produced.t

src/dune_cache/shared.mli

@@ -6,6 +6,14 @@ open Import
     artifacts produced in different workspaces. To restore results from the
     shared cache, Dune copes or hardlinks them into the build directory. *)
 
+module Dynamic_deps : sig
+  (** Dependency manifests use ordinary cache entries so that trimming and
+      concurrent writes follow the same rules as build artifacts. *)
+  val load : rule_digest:Digest.t -> Sexp.t option
+
+  val store : rule_digest:Digest.t -> Sexp.t -> unit
+end
+
 (** Check if the shared cache contains results for a rule and decide whether
       to use these results or rerun the rule for a reproducibility check. *)
 val lookup

src/dune_digest/digest.mli

@@ -117,3 +117,5 @@ val path_with_stats_async
     number of concurrent calls capped by a global throttle so that we do not
     exceed the process's open file descriptor limit. *)
 val file_with_executable_bit : executable:bool -> Path.t -> t Fiber.t
+
+val path_with_executable_bit : executable:bool -> content_digest:t -> t

src/dune_engine/build_system.ml

@@ -644,10 +644,8 @@ module Internal = struct
       in
       let can_go_in_shared_cache =
         props.can_go_in_shared_cache
-        && (not
-              (always_rerun
-               || is_action_dynamic
-               || Action.is_useful_to_memoize action_ast = Clearly_not))
+        && (not (always_rerun || Action.is_useful_to_memoize action_ast = Clearly_not))
+        && ((not is_action_dynamic) || Option.is_none sandbox_mode)
         &&
         match sandbox_mode with
         | Some Patch_back_source_tree ->
@@ -694,17 +692,16 @@ module Internal = struct
           in
           let* produced_targets, dynamic_deps_stages =
             (* Step III. Try to restore artifacts from the shared cache. *)
-            Dune_cache.Shared.lookup ~can_go_in_shared_cache ~rule_digest ~targets
-            >>= function
-            | Some produced_targets ->
-              (* Rules with dynamic deps can't be stored to the shared cache
-                 (see the [is_action_dynamic] check above), so we know this is
-                 not a dynamic action, so returning an empty list is correct.
-                 The lack of information to fill in [dynamic_deps_stages] here
-                 is precisely the reason why we don't store dynamic actions in
-                 the shared cache. *)
-              let dynamic_deps_stages = [] in
-              Fiber.return (produced_targets, dynamic_deps_stages)
+            let* restored =
+              if is_action_dynamic && can_go_in_shared_cache
+              then
+                Rule_cache.Dynamic.lookup ~rule_digest ~targets ~env:props.env ~build_deps
+              else
+                Dune_cache.Shared.lookup ~can_go_in_shared_cache ~rule_digest ~targets
+                >>| Option.map ~f:(fun targets -> targets, [])
+            in
+            match restored with
+            | Some restored -> Fiber.return restored
             | None ->
               (* Step IV. Execute the build action. *)
               let loc = Rule.loc rule in
@@ -719,15 +716,6 @@ module Internal = struct
                   ~sandbox_mode
                   ~targets
               in
-              (* Step V. Examine produced targets and store them to the shared
-                 cache if needed. *)
-              let* produced_targets =
-                Dune_cache.Shared.examine_targets_and_store
-                  ~can_go_in_shared_cache
-                  ~loc
-                  ~rule_digest
-                  ~produced_targets:exec_result.produced_targets
-              in
               let dynamic_deps_stages =
                 List.map
                   exec_result.action_exec_result.dynamic_deps_stages
@@ -737,7 +725,21 @@ module Internal = struct
                       Dep.Facts.digest fact_map d ~env:props.env;
                       Digest.Manual.get d ))
               in
-              Fiber.return (produced_targets, dynamic_deps_stages)
+              let+ produced_targets =
+                (* Step V. Examine produced targets and store them to the shared
+                   cache if needed. *)
+                let cache_key =
+                  if is_action_dynamic && can_go_in_shared_cache
+                  then Rule_cache.Dynamic.store ~rule_digest ~stages:dynamic_deps_stages
+                  else rule_digest
+                in
+                Dune_cache.Shared.examine_targets_and_store
+                  ~can_go_in_shared_cache
+                  ~loc
+                  ~rule_digest:cache_key
+                  ~produced_targets:exec_result.produced_targets
+              in
+              produced_targets, dynamic_deps_stages
           in
           (* We do not include target names into [targets_digest] because they
              are already included into the rule digest. *)