Report remote analysis cache statistics on the BEP.

RELNOTES: None.
PiperOrigin-RevId: 761848466
Change-Id: Ied7a4ea7dcbc6d50a8e666c06247c0b434224d06

Author: lberki

src/main/java/com/google/devtools/build/lib/buildeventstream/proto/build_event_stream.proto

@@ -1363,6 +1363,18 @@ message BuildMetrics {
   message RemoteAnalysisCacheStatistics {
     int64 cache_hits = 1;
     int64 cache_misses = 2;
+    int64 value_store_value_bytes_received = 3;
+    int64 value_store_value_bytes_sent = 4;
+    int64 value_store_key_bytes_sent = 5;
+    int64 value_store_write_ops = 6;
+    int64 value_store_read_ops_successful = 7;
+    int64 value_store_read_ops_not_found = 8;
+    int64 value_store_read_batches = 10;
+    int64 value_store_write_batches = 11;
+    int64 analysis_cache_bytes_received = 12;
+    int64 analysis_cache_key_bytes_sent = 13;
+    int64 analysis_cache_ops = 14;
+    int64 analysis_cache_batches = 15;
   }
   RemoteAnalysisCacheStatistics remote_analysis_cache_statistics = 13;
 }

src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java

@@ -900,6 +900,17 @@ public BuildResult processRequest(
     return result;
   }
 
+  private void reportRemoteAnalysisServiceStats(
+      RemoteAnalysisCachingDependenciesProvider dependenciesProvider) throws InterruptedException {
+    FingerprintValueStore.Stats fvsStats =
+        dependenciesProvider.getFingerprintValueService().getStats();
+    RemoteAnalysisCacheClient.Stats raccStats =
+        dependenciesProvider.getAnalysisCacheClient() == null
+            ? null
+            : dependenciesProvider.getAnalysisCacheClient().getStats();
+    env.getRemoteAnalysisCachingEventListener().recordServiceStats(fvsStats, raccStats);
+  }
+
   /**
    * Handles post-build analysis caching operations.
    *
@@ -919,14 +930,15 @@ private void logAnalysisCachingStatsAndMaybeUploadFrontier(
 
     switch (dependenciesProvider.mode()) {
       case UPLOAD:
-      // fall through
-      case DUMP_UPLOAD_MANIFEST_ONLY:
         uploadFrontier(dependenciesProvider);
+        reportRemoteAnalysisServiceStats(dependenciesProvider);
+        break;
+      case DUMP_UPLOAD_MANIFEST_ONLY:
+        uploadFrontier(dependenciesProvider); // In this case, uploadFrontier() won't upload
         break;
       case DOWNLOAD:
-        reportRemoteAnalysisCachingStats(
-            dependenciesProvider.getFingerprintValueService().getStats(),
-            dependenciesProvider.getAnalysisCacheClient().getStats());
+        reportRemoteAnalysisServiceStats(dependenciesProvider);
+        reportRemoteAnalysisCachingStats();
         env.getSkyframeExecutor()
             .setRemoteAnalysisCachingStateForLatestBuild(
                 env.getRemoteAnalysisCachingEventListener().getRemoteAnalysisCachingState());
@@ -1411,10 +1423,8 @@ private void uploadFrontier(RemoteAnalysisCachingDependenciesProvider dependenci
     }
   }
 
-  private void reportRemoteAnalysisCachingStats(
-      FingerprintValueStore.Stats fvsStats, RemoteAnalysisCacheClient.Stats raccStats) {
+  private void reportRemoteAnalysisCachingStats() {
     var listener = env.getRemoteAnalysisCachingEventListener();
-
     var hitsByFunction = listener.getHitsBySkyFunctionName();
     var missesByFunction = listener.getMissesBySkyFunctionName();
     long totalHits = hitsByFunction.values().stream().mapToLong(AtomicInteger::get).sum();
@@ -1444,15 +1454,25 @@ private void reportRemoteAnalysisCachingStats(
                 })
             .collect(joining(", "));
 
+    FingerprintValueStore.Stats fvsStats = listener.getFingerprintValueStoreStats();
+    RemoteAnalysisCacheClient.Stats raccStats = listener.getRemoteAnalysisCacheStats();
+
+    long bytesReceived = fvsStats.valueBytesReceived();
+    long requests = fvsStats.entriesFound() + fvsStats.entriesNotFound();
+
+    if (raccStats != null) {
+      bytesReceived += raccStats.bytesReceived();
+      requests += raccStats.requestsSent();
+    }
     double overallHitRate = totalRequests == 0 ? 0.0 : (double) totalHits / totalRequests * 100;
     env.getReporter()
         .handle(
             Event.info(
                 String.format(
                     "Remote analysis caching stats: %s bytes received in %s requests, %s/%s cache"
                         + " hits (%.2f%%) [Breakdown: %s]",
-                    raccStats.bytesReceived() + fvsStats.valueBytesReceived(),
-                    raccStats.requestsSent() + fvsStats.entriesFound() + fvsStats.entriesNotFound(),
+                    bytesReceived,
+                    requests,
                     totalHits,
                     totalRequests,
                     overallHitRate,

src/main/java/com/google/devtools/build/lib/metrics/BUILD

@@ -53,6 +53,8 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/skyframe:sky_key_stats",
         "//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_stats",
         "//src/main/java/com/google/devtools/build/lib/skyframe:top_level_status_events",
+        "//src/main/java/com/google/devtools/build/lib/skyframe/serialization",
+        "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:remote_analysis_cache_client",
         "//src/main/java/com/google/devtools/build/lib/worker:worker_process_metrics",
         "//src/main/java/com/google/devtools/build/lib/worker:worker_process_status",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",

src/main/java/com/google/devtools/build/lib/metrics/MetricsCollector.java

@@ -70,6 +70,8 @@
 import com.google.devtools.build.lib.skyframe.SkyframeStats;
 import com.google.devtools.build.lib.skyframe.TopLevelStatusEvents.SomeExecutionStartedEvent;
 import com.google.devtools.build.lib.skyframe.TopLevelStatusEvents.TopLevelTargetPendingExecutionEvent;
+import com.google.devtools.build.lib.skyframe.serialization.FingerprintValueStore;
+import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCacheClient;
 import com.google.devtools.build.lib.worker.WorkerProcessMetrics;
 import com.google.devtools.build.lib.worker.WorkerProcessMetricsCollector;
 import com.google.devtools.build.lib.worker.WorkerProcessStatus;
@@ -331,11 +333,7 @@ private BuildMetrics createBuildMetrics() {
 
     addSkyframeStats(buildGraphMetrics);
 
-    RemoteAnalysisCacheStatistics remoteAnalysisCacheStatistics =
-        RemoteAnalysisCacheStatistics.newBuilder()
-            .setCacheHits(env.getRemoteAnalysisCachingEventListener().getCacheHits().size())
-            .setCacheMisses(env.getRemoteAnalysisCachingEventListener().getCacheMisses().size())
-            .build();
+    RemoteAnalysisCacheStatistics remoteAnalysisCacheStatistics = collectRemoteAnalysisCacheStats();
 
     BuildMetrics.Builder buildMetrics =
         BuildMetrics.newBuilder()
@@ -360,6 +358,39 @@ private BuildMetrics createBuildMetrics() {
     return buildMetrics.build();
   }
 
+  private RemoteAnalysisCacheStatistics collectRemoteAnalysisCacheStats() {
+    RemoteAnalysisCacheStatistics.Builder result =
+        RemoteAnalysisCacheStatistics.newBuilder()
+            .setCacheHits(env.getRemoteAnalysisCachingEventListener().getCacheHits().size())
+            .setCacheMisses(env.getRemoteAnalysisCachingEventListener().getCacheMisses().size());
+
+    FingerprintValueStore.Stats fvsStats =
+        env.getRemoteAnalysisCachingEventListener().getFingerprintValueStoreStats();
+    if (fvsStats != null) {
+      result
+          .setValueStoreValueBytesReceived(fvsStats.valueBytesReceived())
+          .setValueStoreValueBytesSent(fvsStats.valueBytesSent())
+          .setValueStoreKeyBytesSent(fvsStats.keyBytesSent())
+          .setValueStoreWriteOps(fvsStats.entriesWritten())
+          .setValueStoreReadOpsSuccessful(fvsStats.entriesFound())
+          .setValueStoreReadOpsNotFound(fvsStats.entriesNotFound())
+          .setValueStoreReadBatches(fvsStats.getBatches())
+          .setValueStoreWriteBatches(fvsStats.setBatches());
+    }
+
+    RemoteAnalysisCacheClient.Stats raccStats =
+        env.getRemoteAnalysisCachingEventListener().getRemoteAnalysisCacheStats();
+    if (raccStats != null) {
+      result
+          .setAnalysisCacheBytesReceived(raccStats.bytesReceived())
+          .setAnalysisCacheKeyBytesSent(raccStats.bytesSent())
+          .setAnalysisCacheOps(raccStats.requestsSent())
+          .setAnalysisCacheBatches(raccStats.batches());
+    }
+
+    return result.build();
+  }
+
   private ActionData buildActionData(ActionStats actionStats) {
     NanosToMillisSinceEpochConverter nanosToMillisSinceEpochConverter =
         BlazeClock.createNanosToMillisSinceEpochConverter();

src/main/java/com/google/devtools/build/lib/skyframe/serialization/FingerprintValueStore.java

@@ -32,10 +32,12 @@ record Stats(
       long keyBytesSent,
       long entriesWritten,
       long entriesFound,
-      long entriesNotFound) {}
+      long entriesNotFound,
+      long getBatches,
+      long setBatches) {}
 
   default Stats getStats() {
-    return new Stats(0, 0, 0, 0, 0, 0);
+    return new Stats(0, 0, 0, 0, 0, 0, 0, 0);
   }
 
   /**

src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/BUILD

@@ -15,11 +15,13 @@ java_library(
     name = "event_listener",
     srcs = ["RemoteAnalysisCachingEventListener.java"],
     deps = [
+        ":remote_analysis_cache_client",
         ":remote_analysis_caching_state",
         "//src/main/java/com/google/devtools/build/lib/concurrent",
         "//src/main/java/com/google/devtools/build/lib/skyframe/serialization",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
         "//third_party:guava",
+        "//third_party:jsr305",
     ],
 )
 

src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/RemoteAnalysisCacheClient.java

@@ -20,7 +20,7 @@
 /** Interface to the remote analysis cache. */
 public interface RemoteAnalysisCacheClient {
   /** Usage statistics. */
-  record Stats(long bytesSent, long bytesReceived, long requestsSent) {}
+  record Stats(long bytesSent, long bytesReceived, long requestsSent, long batches) {}
 
   /** Looks up an entry in the remote analysis cache based on a serialized key. */
   ListenableFuture<ByteString> lookup(ByteString key);

src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/RemoteAnalysisCachingEventListener.java

@@ -23,6 +23,7 @@
 import com.google.common.eventbus.Subscribe;
 import com.google.devtools.build.lib.concurrent.ThreadSafety;
 import com.google.devtools.build.lib.concurrent.ThreadSafety.ThreadSafe;
+import com.google.devtools.build.lib.skyframe.serialization.FingerprintValueStore;
 import com.google.devtools.build.lib.skyframe.serialization.SerializationException;
 import com.google.devtools.build.lib.skyframe.serialization.SkyValueRetriever.FrontierNodeVersion;
 import com.google.devtools.build.lib.skyframe.serialization.SkyValueRetriever.NoCachedData;
@@ -35,6 +36,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.Nullable;
 
 /** An {@link com.google.common.eventbus.EventBus} listener for remote analysis caching events. */
 @ThreadSafety.ThreadSafe
@@ -61,6 +63,9 @@ public record SerializedNodeEvent(SkyKey key) {
 
   private final AtomicReference<FrontierNodeVersion> skyValueVersion = new AtomicReference<>();
 
+  @Nullable private FingerprintValueStore.Stats fingerprintValueStoreStats;
+  @Nullable private RemoteAnalysisCacheClient.Stats remoteAnalysisCacheStats;
+
   @Subscribe
   @AllowConcurrentEvents
   @SuppressWarnings("unused")
@@ -92,6 +97,21 @@ public Set<SkyKey> getCacheMisses() {
     return ImmutableSet.copyOf(cacheMisses);
   }
 
+  public void recordServiceStats(
+      @Nullable FingerprintValueStore.Stats fvsStats,
+      @Nullable RemoteAnalysisCacheClient.Stats raccStats) {
+    fingerprintValueStoreStats = fvsStats;
+    remoteAnalysisCacheStats = raccStats;
+  }
+
+  public FingerprintValueStore.Stats getFingerprintValueStoreStats() {
+    return fingerprintValueStoreStats;
+  }
+
+  public RemoteAnalysisCacheClient.Stats getRemoteAnalysisCacheStats() {
+    return remoteAnalysisCacheStats;
+  }
+
   @ThreadSafe
   public void recordRetrievalResult(RetrievalResult result, SkyKey key) {
     if (result instanceof Restart) {