Repo contents cache GC

Very simple GC implementation for the repo contents cache. Entry access is logged by "touching" the recorded inputs file. GC tasks then delete old entries that haven't been accessed in `--repo_contents_cache_gc_max_age` time.

Work towards #12227.

Closes #26080.

PiperOrigin-RevId: 761930313
Change-Id: I6c0b92771f57d9949380fb08698385c4a96fe7d4

Author: Wyverald

src/main/java/com/google/devtools/build/lib/bazel/BUILD

@@ -47,6 +47,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/pkgcache",
+        "//src/main/java/com/google/devtools/build/lib/profiler",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/local_repository_rule",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/new_local_repository_function",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/new_local_repository_rule",

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -81,6 +81,9 @@
 import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.pkgcache.PackageOptions;
+import com.google.devtools.build.lib.profiler.Profiler;
+import com.google.devtools.build.lib.profiler.ProfilerTask;
+import com.google.devtools.build.lib.profiler.SilentCloseable;
 import com.google.devtools.build.lib.rules.repository.LocalRepositoryFunction;
 import com.google.devtools.build.lib.rules.repository.LocalRepositoryRule;
 import com.google.devtools.build.lib.rules.repository.NewLocalRepositoryFunction;
@@ -382,6 +385,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
         // invocation, as is often the case with the repo contents cache.
         // TODO: wyv@ - This is a crude check that disables some use cases (such as when the output
         //   base itself is inside the main repo). Investigate a better check.
+        repositoryCache.getRepoContentsCache().setPath(null);
         throw new AbruptExitException(
             detailedExitCode(
                 """
@@ -392,6 +396,26 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
                     .formatted(repoContentsCachePath, env.getWorkspace()),
                 Code.BAD_REPO_CONTENTS_CACHE));
       }
+      if (repositoryCache.getRepoContentsCache().isEnabled()) {
+        try (SilentCloseable c =
+            Profiler.instance()
+                .profile(ProfilerTask.REPO_CACHE_GC_WAIT, "waiting to acquire repo cache lock")) {
+          repositoryCache.getRepoContentsCache().acquireSharedLock();
+        } catch (IOException e) {
+          throw new AbruptExitException(
+              detailedExitCode(
+                  "could not acquire lock on repo contents cache", Code.BAD_REPO_CONTENTS_CACHE),
+              e);
+        }
+        if (!repoOptions.repoContentsCacheGcMaxAge.isZero()) {
+          env.addIdleTask(
+              repositoryCache
+                  .getRepoContentsCache()
+                  .createGcIdleTask(
+                      repoOptions.repoContentsCacheGcMaxAge,
+                      repoOptions.repoContentsCacheGcIdleDelay));
+        }
+      }
 
       try {
         downloadManager.setNetrcCreds(
@@ -648,6 +672,20 @@ private Path toPath(PathFragment path, CommandEnvironment env) {
     return env.getBlazeWorkspace().getWorkspace().getRelative(path);
   }
 
+  @Override
+  public void afterCommand() throws AbruptExitException {
+    if (repositoryCache.getRepoContentsCache().isEnabled()) {
+      try {
+        repositoryCache.getRepoContentsCache().releaseSharedLock();
+      } catch (IOException e) {
+        throw new AbruptExitException(
+            detailedExitCode(
+                "could not release lock on repo contents cache", Code.BAD_REPO_CONTENTS_CACHE),
+            e);
+      }
+    }
+  }
+
   @Override
   public ImmutableList<Injected> getPrecomputedValues() {
     Instant now = clock.now();

src/main/java/com/google/devtools/build/lib/bazel/repository/RepositoryOptions.java

@@ -19,13 +19,15 @@
 import com.google.devtools.build.lib.util.OptionsUtils;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.common.options.Converter;
+import com.google.devtools.common.options.Converters.DurationConverter;
 import com.google.devtools.common.options.EnumConverter;
 import com.google.devtools.common.options.Option;
 import com.google.devtools.common.options.OptionDocumentationCategory;
 import com.google.devtools.common.options.OptionEffectTag;
 import com.google.devtools.common.options.OptionMetadataTag;
 import com.google.devtools.common.options.OptionsBase;
 import com.google.devtools.common.options.OptionsParsingException;
+import java.time.Duration;
 import java.util.List;
 import net.starlark.java.eval.EvalException;
 
@@ -64,6 +66,32 @@ public class RepositoryOptions extends OptionsBase {
           """)
   public PathFragment repoContentsCache;
 
+  @Option(
+      name = "repo_contents_cache_gc_max_age",
+      defaultValue = "14d",
+      documentationCategory = OptionDocumentationCategory.BAZEL_CLIENT_OPTIONS,
+      effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION},
+      converter = DurationConverter.class,
+      help =
+          """
+          Specifies the amount of time an entry in the repo contents cache can stay unused before \
+          it's garbage collected. If set to zero, garbage collection is disabled.
+          """)
+  public Duration repoContentsCacheGcMaxAge;
+
+  @Option(
+      name = "repo_contents_cache_gc_idle_delay",
+      defaultValue = "5m",
+      documentationCategory = OptionDocumentationCategory.BAZEL_CLIENT_OPTIONS,
+      effectTags = {OptionEffectTag.BAZEL_INTERNAL_CONFIGURATION},
+      converter = DurationConverter.class,
+      help =
+          """
+          Specifies the amount of time the server must remain idle before garbage collection happens
+          to the repo contents cache.
+          """)
+  public Duration repoContentsCacheGcIdleDelay;
+
   @Option(
       name = "registry",
       defaultValue = "null",

src/main/java/com/google/devtools/build/lib/bazel/repository/cache/BUILD

@@ -21,6 +21,7 @@ java_library(
         "RepositoryCache.java",
     ],
     deps = [
+        "//src/main/java/com/google/devtools/build/lib/server:idle_task",
         "//src/main/java/com/google/devtools/build/lib/util:file_system_lock",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs/bazel",

src/main/java/com/google/devtools/build/lib/bazel/repository/cache/RepoContentsCache.java

@@ -18,22 +18,65 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
+import com.google.devtools.build.lib.server.IdleTask;
+import com.google.devtools.build.lib.server.IdleTaskException;
 import com.google.devtools.build.lib.util.FileSystemLock;
 import com.google.devtools.build.lib.util.FileSystemLock.LockMode;
+import com.google.devtools.build.lib.vfs.Dirent;
 import com.google.devtools.build.lib.vfs.FileSystemUtils;
 import com.google.devtools.build.lib.vfs.Path;
+import com.google.devtools.build.lib.vfs.Symlinks;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.Comparator;
+import java.util.UUID;
 import javax.annotation.Nullable;
 
-/** A cache directory that stores the contents of fetched repos across different workspaces. */
-public class RepoContentsCache {
+/**
+ * A cache directory that stores the contents of fetched repos across different workspaces.
+ *
+ * <p>The repo contents cache is laid out in two layers. The first layer is a lookup by "predeclared
+ * inputs hash", which is defined as the hash of all predeclared inputs of a repo (such as
+ * transitive bzl digest, repo attrs, starlark semantics, etc). Each distinct predeclared inputs
+ * hash is its own entry directory in the first layer.
+ *
+ * <p>Inside each entry directory are pairs of directories and files {@code <N, N.recorded_inputs>}
+ * where {@code N} is an integer. The file {@code N.recorded_inputs} contains the recorded inputs
+ * and their values of a cached repo, and the directory {@code N} contains the cached repo contents.
+ * There is also a file named {@code counter} that stores the next available {@code N} for this
+ * entry directory, and a file named {@code lock} to ensure exclusive access to the {@code counter}
+ * file.
+ *
+ * <p>On a cache hit (that is, the predeclared inputs hash matches, and recorded inputs are
+ * up-to-date), the recorded inputs file has its mtime updated. Cached repos whose recorded inputs
+ * file is older than {@code --repo_contents_cache_gc_max_age} are garbage collected.
+ */
+public final class RepoContentsCache {
   public static final String RECORDED_INPUTS_SUFFIX = ".recorded_inputs";
 
-  @Nullable private Path path;
+  /**
+   * The path to a "lock" file, relative to the root of the repo contents cache. While a shared lock
+   * is held, no garbage collection should happen. While an exclusive lock is held, no reads should
+   * happen.
+   */
+  public static final String LOCK_PATH = "gc_lock";
+
+  /**
+   * The path to a trash directory relative to the root of the repo contents cache.
+   *
+   * <p>Since deleting entire directories could take a bit of time, we create a trash directory
+   * where we move the garbage directories to (which should be very fast). Then we can delete this
+   * trash directory altogether at the end. This makes the GC process safe against being interrupted
+   * in the middle (any undeleted trash will get deleted by the next GC). Also be sure to name this
+   * trashDir something that couldn't ever be a predeclared inputs hash (starting with an underscore
+   * should suffice).
+   */
+  public static final String TRASH_PATH = "_trash";
 
-  // TODO: wyv@ - implement garbage collection
+  @Nullable private Path path;
+  @Nullable private FileSystemLock sharedLock;
 
   public void setPath(@Nullable Path path) {
     this.path = path;
@@ -58,6 +101,15 @@ private static CandidateRepo fromRecordedInputsFile(Path recordedInputsFile) {
       return new CandidateRepo(
           recordedInputsFile, recordedInputsFile.replaceName(contentsDirBaseName));
     }
+
+    /** Updates the mtime of the recorded inputs file, to delay GC for this entry. */
+    public void touch() {
+      try {
+        recordedInputsFile.setLastModifiedTime(Path.NOW_SENTINEL_TIME);
+      } catch (IOException e) {
+        // swallow the exception. it's not a huge deal.
+      }
+    }
   }
 
   /** Returns the list of candidate repos for the given predeclared input hash. */
@@ -80,8 +132,19 @@ public ImmutableList<CandidateRepo> getCandidateRepos(String predeclaredInputHas
     }
   }
 
-  /** Moves a freshly fetched repo into the contents cache. */
-  public void moveToCache(
+  private Path ensureTrashDir() throws IOException {
+    Preconditions.checkState(path != null);
+    Path trashDir = path.getChild(TRASH_PATH);
+    trashDir.createDirectoryAndParents();
+    return trashDir;
+  }
+
+  /**
+   * Moves a freshly fetched repo into the contents cache.
+   *
+   * @return the repo dir in the contents cache.
+   */
+  public Path moveToCache(
       Path fetchedRepoDir, Path fetchedRepoMarkerFile, String predeclaredInputHash)
       throws IOException {
     Preconditions.checkState(path != null);
@@ -98,7 +161,7 @@ public void moveToCache(
     cacheRepoDir.createDirectoryAndParents();
     // Move the fetched marker file to a temp location, so that if following operations fail, both
     // the fetched repo and the cache locations are considered out-of-date.
-    Path temporaryMarker = entryDir.getChild(counter + ".temp_recorded_inputs");
+    Path temporaryMarker = ensureTrashDir().getChild(UUID.randomUUID().toString());
     FileSystemUtils.moveFile(fetchedRepoMarkerFile, temporaryMarker);
     // Now perform the move, and afterwards, restore the marker file.
     try {
@@ -110,6 +173,7 @@ public void moveToCache(
     // Set up a symlink at the original fetched repo dir path.
     fetchedRepoDir.deleteTree();
     FileSystemUtils.ensureSymbolicLink(fetchedRepoDir, cacheRepoDir);
+    return cacheRepoDir;
   }
 
   private static String getNextCounterInDir(Path entryDir) throws IOException {
@@ -128,4 +192,80 @@ private static String getNextCounterInDir(Path entryDir) throws IOException {
       return counter;
     }
   }
+
+  public void acquireSharedLock() throws IOException {
+    Preconditions.checkState(path != null);
+    Preconditions.checkState(sharedLock == null, "this process already has the shared lock");
+    sharedLock = FileSystemLock.get(path.getRelative(LOCK_PATH), LockMode.SHARED);
+  }
+
+  public void releaseSharedLock() throws IOException {
+    Preconditions.checkState(sharedLock != null);
+    sharedLock.close();
+    sharedLock = null;
+  }
+
+  /**
+   * Creates a garbage collection {@link IdleTask} that deletes cached repos who are last accessed
+   * more than {@code maxAge} ago, with an idle delay of {@code idleDelay}.
+   */
+  public IdleTask createGcIdleTask(Duration maxAge, Duration idleDelay) {
+    Preconditions.checkState(path != null);
+    return new IdleTask() {
+      @Override
+      public String displayName() {
+        return "Repo contents cache garbage collection";
+      }
+
+      @Override
+      public Duration delay() {
+        return idleDelay;
+      }
+
+      @Override
+      public void run() throws InterruptedException, IdleTaskException {
+        try {
+          Preconditions.checkState(path != null);
+          // If we can't grab the lock, abort GC. Someone will come along later.
+          try (var lock = FileSystemLock.tryGet(path.getRelative(LOCK_PATH), LockMode.EXCLUSIVE)) {
+            runGc(maxAge);
+          }
+          // Empty the trash dir outside the lock. No one is reading from these files, so it should
+          // be safe. At worst, multiple servers performing GC will try to delete the same files,
+          // but whatever.
+          path.getChild(TRASH_PATH).deleteTreesBelow();
+        } catch (IOException e) {
+          throw new IdleTaskException(e);
+        }
+      }
+    };
+  }
+
+  private void runGc(Duration maxAge) throws InterruptedException, IOException {
+    path.setLastModifiedTime(Path.NOW_SENTINEL_TIME);
+    Instant cutoff = Instant.ofEpochMilli(path.getLastModifiedTime()).minus(maxAge);
+    Path trashDir = ensureTrashDir();
+
+    for (Dirent dirent : path.readdir(Symlinks.NOFOLLOW)) {
+      if (dirent.getType() != Dirent.Type.DIRECTORY || dirent.getName().equals(TRASH_PATH)) {
+        continue;
+      }
+      for (Path recordedInputsFile : path.getChild(dirent.getName()).getDirectoryEntries()) {
+        if (!recordedInputsFile.getBaseName().endsWith(RECORDED_INPUTS_SUFFIX)) {
+          continue;
+        }
+        if (Thread.interrupted()) {
+          throw new InterruptedException();
+        }
+
+        if (Instant.ofEpochMilli(recordedInputsFile.getLastModifiedTime()).isBefore(cutoff)) {
+          // Sorry buddy, you're out.
+          recordedInputsFile.delete();
+          var repoDir = CandidateRepo.fromRecordedInputsFile(recordedInputsFile).contentsDir;
+          // Use a UUID to avoid clashes.
+          repoDir.renameTo(trashDir.getChild(UUID.randomUUID().toString()));
+        }
+      }
+    }
+  }
 }

src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkRepositoryFunction.java

@@ -123,6 +123,10 @@ public void reportSkyframeRestart(Environment env, RepositoryName repoName) {
     }
   }
 
+  private static class State extends WorkerSkyKeyComputeState<FetchResult> {
+    @Nullable FetchResult result;
+  }
+
   private record FetchArgs(
       Rule rule, Path outputDirectory, BlazeDirectories directories, Environment env, SkyKey key) {
     FetchArgs toWorkerArgs(Environment env) {
@@ -141,15 +145,23 @@ public FetchResult fetch(
     }
     // See below (the `catch CancellationException` clause) for why there's a `while` loop here.
     while (true) {
-      var state = env.getState(WorkerSkyKeyComputeState<FetchResult>::new);
+      var state = env.getState(State::new);
+      if (state.result != null) {
+        // Escape early if we've already finished fetching once. This can happen if
+        // RepositoryDelegatorFunction triggers a Skyframe restart _after_
+        // StarlarkRepositoryFunction#fetch is finished.
+        return state.result;
+      }
       try {
-        return state.startOrContinueWork(
-            env,
-            "starlark-repository-" + rule.getName(),
-            (workerEnv) -> {
-              setupRepoRoot(outputDirectory);
-              return fetchInternal(args.toWorkerArgs(workerEnv));
-            });
+        state.result =
+            state.startOrContinueWork(
+                env,
+                "starlark-repository-" + rule.getName(),
+                (workerEnv) -> {
+                  setupRepoRoot(outputDirectory);
+                  return fetchInternal(args.toWorkerArgs(workerEnv));
+                });
+        return state.result;
       } catch (ExecutionException e) {
         Throwables.throwIfInstanceOf(e.getCause(), RepositoryFunctionException.class);
         Throwables.throwIfInstanceOf(e.getCause(), InterruptedException.class);

src/main/java/com/google/devtools/build/lib/profiler/ProfilerTask.java

@@ -88,6 +88,7 @@ public enum ProfilerTask {
   DYNAMIC_LOCK("Acquiring dynamic execution output lock", Threshold.FIFTY_MILLIS),
   REPOSITORY_FETCH("Fetching repository"),
   REPOSITORY_VENDOR("Vendoring repository"),
+  REPO_CACHE_GC_WAIT("blocked on repo contents cache GC", Threshold.TEN_MILLIS),
   SPAWN_LOG("logging spawn", Threshold.TEN_MILLIS),
 
   UNKNOWN("Unknown event");