Skip to content

Commit a7dc3eb

Browse files
lberkicopybara-github
lberki
authored and
copybara-github
committed
Record network statistics for the analysis cache client.
RELNOTES: None. PiperOrigin-RevId: 761836388 Change-Id: I8832af5dd11b37f841e18cfa17963871f7fba35d
1 parent 190b863 commit a7dc3eb

File tree

13 files changed

+101
-64
lines changed

13 files changed

+101
-64
lines changed

‎src/main/java/com/google/devtools/build/lib/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -488,6 +488,7 @@ java_library(
488488
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:event_listener",
489489
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:frontier_serializer",
490490
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:options",
491+
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:remote_analysis_cache_client",
491492
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:remote_analysis_caching_services_supplier",
492493
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:remote_analysis_caching_state",
493494
"//src/main/java/com/google/devtools/build/lib/unsafe:string",

‎src/main/java/com/google/devtools/build/lib/buildtool/BuildTool.java

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@
6767
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
6868
import com.google.devtools.build.lib.cmdline.TargetParsingException;
6969
import com.google.devtools.build.lib.collect.PathFragmentPrefixTrie;
70-
import com.google.devtools.build.lib.concurrent.RequestBatcher;
7170
import com.google.devtools.build.lib.events.Event;
7271
import com.google.devtools.build.lib.events.ExtendedEventHandler;
7372
import com.google.devtools.build.lib.events.OutputFilter;
@@ -114,6 +113,7 @@
114113
import com.google.devtools.build.lib.skyframe.serialization.analysis.AnalysisCacheInvalidator;
115114
import com.google.devtools.build.lib.skyframe.serialization.analysis.ClientId;
116115
import com.google.devtools.build.lib.skyframe.serialization.analysis.FrontierSerializer;
116+
import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCacheClient;
117117
import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingDependenciesProvider;
118118
import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingEventListener;
119119
import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingOptions;
@@ -139,7 +139,6 @@
139139
import com.google.devtools.common.options.OptionsParsingException;
140140
import com.google.devtools.common.options.OptionsParsingResult;
141141
import com.google.devtools.common.options.RegexPatternOption;
142-
import com.google.protobuf.ByteString;
143142
import java.io.BufferedOutputStream;
144143
import java.io.IOException;
145144
import java.io.OutputStream;
@@ -926,7 +925,8 @@ private void logAnalysisCachingStatsAndMaybeUploadFrontier(
926925
break;
927926
case DOWNLOAD:
928927
reportRemoteAnalysisCachingStats(
929-
dependenciesProvider.getFingerprintValueService().getStats());
928+
dependenciesProvider.getFingerprintValueService().getStats(),
929+
dependenciesProvider.getAnalysisCacheClient().getStats());
930930
env.getSkyframeExecutor()
931931
.setRemoteAnalysisCachingStateForLatestBuild(
932932
env.getRemoteAnalysisCachingEventListener().getRemoteAnalysisCachingState());
@@ -1093,7 +1093,7 @@ private static class RemoteAnalysisCachingDependenciesProviderImpl
10931093

10941094
private final Future<ObjectCodecs> objectCodecsFuture;
10951095
private final Future<FingerprintValueService> fingerprintValueServiceFuture;
1096-
@Nullable private final Future<RequestBatcher<ByteString, ByteString>> analysisCacheClient;
1096+
@Nullable private final Future<RemoteAnalysisCacheClient> analysisCacheClient;
10971097
@Nullable private volatile AnalysisCacheInvalidator analysisCacheInvalidator;
10981098

10991099
// Non-final because the top level BuildConfigurationValue is determined just before analysis
@@ -1330,7 +1330,7 @@ public FingerprintValueService getFingerprintValueService() throws InterruptedEx
13301330

13311331
@Override
13321332
@Nullable
1333-
public RequestBatcher<ByteString, ByteString> getAnalysisCacheClient() {
1333+
public RemoteAnalysisCacheClient getAnalysisCacheClient() {
13341334
if (analysisCacheClient == null) {
13351335
return null;
13361336
}
@@ -1375,7 +1375,7 @@ private AnalysisCacheInvalidator getAnalysisCacheInvalidator() throws Interrupte
13751375
if (localRef == null) {
13761376
ObjectCodecs codecs;
13771377
FingerprintValueService fingerprintService;
1378-
RequestBatcher<ByteString, ByteString> client;
1378+
RemoteAnalysisCacheClient client;
13791379
try (SilentCloseable unused =
13801380
Profiler.instance().profile("initializeInvalidationLookupDeps")) {
13811381
client = getAnalysisCacheClient();
@@ -1411,7 +1411,8 @@ private void uploadFrontier(RemoteAnalysisCachingDependenciesProvider dependenci
14111411
}
14121412
}
14131413

1414-
private void reportRemoteAnalysisCachingStats(FingerprintValueStore.Stats stats) {
1414+
private void reportRemoteAnalysisCachingStats(
1415+
FingerprintValueStore.Stats fvsStats, RemoteAnalysisCacheClient.Stats raccStats) {
14151416
var listener = env.getRemoteAnalysisCachingEventListener();
14161417

14171418
var hitsByFunction = listener.getHitsBySkyFunctionName();
@@ -1448,10 +1449,10 @@ private void reportRemoteAnalysisCachingStats(FingerprintValueStore.Stats stats)
14481449
.handle(
14491450
Event.info(
14501451
String.format(
1451-
"Remote analysis caching stats: %s bytes and %s entries received, %s/%s cache"
1452+
"Remote analysis caching stats: %s bytes received in %s requests, %s/%s cache"
14521453
+ " hits (%.2f%%) [Breakdown: %s]",
1453-
stats.valueBytesReceived(),
1454-
stats.entriesFound(),
1454+
raccStats.bytesReceived() + fvsStats.valueBytesReceived(),
1455+
raccStats.requestsSent() + fvsStats.entriesFound() + fvsStats.entriesNotFound(),
14551456
totalHits,
14561457
totalRequests,
14571458
overallHitRate,

‎src/main/java/com/google/devtools/build/lib/skyframe/serialization/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ java_library(
8686
"//src/main/java/com/google/devtools/build/lib/bugreport",
8787
"//src/main/java/com/google/devtools/build/lib/concurrent",
8888
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:client_id",
89+
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:remote_analysis_cache_client",
8990
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:registered-singleton",
9091
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:serialization-constant",
9192
"//src/main/java/com/google/devtools/build/lib/unsafe:string",

‎src/main/java/com/google/devtools/build/lib/skyframe/serialization/SkyValueRetriever.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,10 @@
2424
import com.google.common.primitives.Bytes;
2525
import com.google.common.primitives.Longs;
2626
import com.google.common.util.concurrent.ListenableFuture;
27-
import com.google.devtools.build.lib.concurrent.RequestBatcher;
2827
import com.google.devtools.build.lib.skyframe.serialization.FingerprintValueStore.MissingFingerprintValueException;
2928
import com.google.devtools.build.lib.skyframe.serialization.SharedValueDeserializationContext.StateEvictedException;
3029
import com.google.devtools.build.lib.skyframe.serialization.analysis.ClientId;
30+
import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCacheClient;
3131
import com.google.devtools.build.lib.skyframe.serialization.proto.DataType;
3232
import com.google.devtools.build.skyframe.IntVersion;
3333
import com.google.devtools.build.skyframe.SkyFunction.Environment;
@@ -211,7 +211,7 @@ public static RetrievalResult tryRetrieve(
211211
DependOnFutureShim futuresShim,
212212
ObjectCodecs codecs,
213213
FingerprintValueService fingerprintValueService,
214-
@Nullable RequestBatcher<ByteString, ByteString> analysisCacheClient,
214+
@Nullable RemoteAnalysisCacheClient analysisCacheClient,
215215
SkyKey key,
216216
SerializationStateProvider stateProvider,
217217
FrontierNodeVersion frontierNodeVersion)
@@ -238,7 +238,7 @@ public static RetrievalResult tryRetrieve(
238238
responseFuture = futureValueBytes;
239239
} else {
240240
ListenableFuture<ByteString> futureResponseBytes =
241-
analysisCacheClient.submit(ByteString.copyFrom(cacheKey.toBytes()));
241+
analysisCacheClient.lookup(ByteString.copyFrom(cacheKey.toBytes()));
242242

243243
nextState = new WaitingForCacheServiceResponse(futureResponseBytes);
244244
responseFuture = futureResponseBytes;

‎src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/AnalysisCacheInvalidator.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@
2626
import com.google.common.flogger.GoogleLogger;
2727
import com.google.common.util.concurrent.Futures;
2828
import com.google.common.util.concurrent.ListenableFuture;
29-
import com.google.devtools.build.lib.concurrent.RequestBatcher;
3029
import com.google.devtools.build.lib.events.Event;
3130
import com.google.devtools.build.lib.events.ExtendedEventHandler;
3231
import com.google.devtools.build.lib.profiler.Profiler;
@@ -49,14 +48,14 @@ public final class AnalysisCacheInvalidator {
4948

5049
private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
5150

52-
private final RequestBatcher<ByteString, ByteString> analysisCacheClient;
51+
private final RemoteAnalysisCacheClient analysisCacheClient;
5352
private final ObjectCodecs codecs;
5453
private final FingerprintValueService fingerprintService;
5554
private final ExtendedEventHandler eventHandler;
5655
private final FrontierNodeVersion currentVersion;
5756

5857
public AnalysisCacheInvalidator(
59-
RequestBatcher<ByteString, ByteString> analysisCacheClient,
58+
RemoteAnalysisCacheClient analysisCacheClient,
6059
ObjectCodecs objectCodecs,
6160
FingerprintValueService fingerprintValueService,
6261
FrontierNodeVersion currentVersion,
@@ -150,7 +149,7 @@ private ListenableFuture<Optional<SkyKey>> submitInvalidationLookup(SkyKey key)
150149

151150
// 2. Submit the fingerprint to the analysis cache service
152151
ListenableFuture<ByteString> responseFuture =
153-
analysisCacheClient.submit(ByteString.copyFrom(cacheKey.toBytes()));
152+
analysisCacheClient.lookup(ByteString.copyFrom(cacheKey.toBytes()));
154153

155154
// 3. Transform result to return keys that should be invalidated (i.e.
156155
// empty response, cache miss)

‎src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/BUILD

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,12 @@ java_library(
2828
srcs = ["RemoteAnalysisCachingDependenciesProvider.java"],
2929
deps = [
3030
":options",
31+
":remote_analysis_cache_client",
3132
":remote_analysis_caching_state",
3233
"//src/main/java/com/google/devtools/build/lib/cmdline",
33-
"//src/main/java/com/google/devtools/build/lib/concurrent",
3434
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization",
3535
"//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
3636
"//third_party:guava",
37-
"@com_google_protobuf//:protobuf_java",
3837
],
3938
)
4039

@@ -222,17 +221,25 @@ java_library(
222221
],
223222
)
224223

224+
java_library(
225+
name = "remote_analysis_cache_client",
226+
srcs = ["RemoteAnalysisCacheClient.java"],
227+
deps = [
228+
"//third_party:guava",
229+
"@com_google_protobuf//:protobuf_java",
230+
],
231+
)
232+
225233
java_library(
226234
name = "remote_analysis_caching_services_supplier",
227235
srcs = ["RemoteAnalysisCachingServicesSupplier.java"],
228236
deps = [
229237
":client_id",
230238
":options",
231-
"//src/main/java/com/google/devtools/build/lib/concurrent",
239+
":remote_analysis_cache_client",
232240
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization",
233241
"//third_party:guava",
234242
"//third_party:jsr305",
235-
"@com_google_protobuf//:protobuf_java",
236243
],
237244
)
238245

@@ -250,8 +257,8 @@ java_library(
250257
name = "analysis_cache_invalidator",
251258
srcs = ["AnalysisCacheInvalidator.java"],
252259
deps = [
260+
":remote_analysis_cache_client",
253261
":remote_analysis_caching_state",
254-
"//src/main/java/com/google/devtools/build/lib/concurrent",
255262
"//src/main/java/com/google/devtools/build/lib/events",
256263
"//src/main/java/com/google/devtools/build/lib/profiler",
257264
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization",

‎src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/RemoteAnalysisCacheClient.java

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
// Copyright 2025 The Bazel Authors. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package com.google.devtools.build.lib.skyframe.serialization.analysis;
16+
17+
import com.google.common.util.concurrent.ListenableFuture;
18+
import com.google.protobuf.ByteString;
19+
20+
/** Interface to the remote analysis cache. */
21+
public interface RemoteAnalysisCacheClient {
22+
/** Usage statistics. */
23+
record Stats(long bytesSent, long bytesReceived, long requestsSent) {}
24+
25+
/** Looks up an entry in the remote analysis cache based on a serialized key. */
26+
ListenableFuture<ByteString> lookup(ByteString key);
27+
28+
/** Returns the usage statistics. */
29+
Stats getStats();
30+
}

‎src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/RemoteAnalysisCachingDependenciesProvider.java

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,15 +15,13 @@
1515

1616
import com.google.common.collect.ImmutableSet;
1717
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
18-
import com.google.devtools.build.lib.concurrent.RequestBatcher;
1918
import com.google.devtools.build.lib.skyframe.serialization.FingerprintValueService;
2019
import com.google.devtools.build.lib.skyframe.serialization.ObjectCodecs;
2120
import com.google.devtools.build.lib.skyframe.serialization.SerializationException;
2221
import com.google.devtools.build.lib.skyframe.serialization.SkyValueRetriever.FrontierNodeVersion;
2322
import com.google.devtools.build.lib.skyframe.serialization.SkyValueRetriever.RetrievalResult;
2423
import com.google.devtools.build.lib.skyframe.serialization.analysis.RemoteAnalysisCachingOptions.RemoteAnalysisCacheMode;
2524
import com.google.devtools.build.skyframe.SkyKey;
26-
import com.google.protobuf.ByteString;
2725

2826
/**
2927
* An interface providing the functionalities used for analysis caching serialization and
@@ -59,7 +57,7 @@ default boolean isRemoteFetchEnabled() {
5957
/** Returns the {@link FingerprintValueService} implementation. */
6058
FingerprintValueService getFingerprintValueService() throws InterruptedException;
6159

62-
RequestBatcher<ByteString, ByteString> getAnalysisCacheClient();
60+
RemoteAnalysisCacheClient getAnalysisCacheClient();
6361

6462
void recordRetrievalResult(RetrievalResult retrievalResult, SkyKey key);
6563

@@ -113,7 +111,7 @@ public FingerprintValueService getFingerprintValueService() {
113111
}
114112

115113
@Override
116-
public RequestBatcher<ByteString, ByteString> getAnalysisCacheClient() {
114+
public RemoteAnalysisCacheClient getAnalysisCacheClient() {
117115
throw new UnsupportedOperationException();
118116
}
119117

‎src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis/RemoteAnalysisCachingServicesSupplier.java

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,7 @@
1414
package com.google.devtools.build.lib.skyframe.serialization.analysis;
1515

1616
import com.google.common.util.concurrent.ListenableFuture;
17-
import com.google.devtools.build.lib.concurrent.RequestBatcher;
1817
import com.google.devtools.build.lib.skyframe.serialization.FingerprintValueService;
19-
import com.google.protobuf.ByteString;
2018
import javax.annotation.Nullable;
2119

2220
/**
@@ -56,7 +54,7 @@ default void configure(RemoteAnalysisCachingOptions cachingOptions, @Nullable Cl
5654
* <p>This may entail I/O so it is wrapped in a future.
5755
*/
5856
@Nullable // null if frontier-style invalidation is used instead of the cache service
59-
default ListenableFuture<RequestBatcher<ByteString, ByteString>> getAnalysisCacheClient() {
57+
default ListenableFuture<RemoteAnalysisCacheClient> getAnalysisCacheClient() {
6058
return null;
6159
}
6260
}

‎src/test/java/com/google/devtools/build/lib/skyframe/serialization/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -437,6 +437,7 @@ java_library(
437437
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization",
438438
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization:visible-for-serialization",
439439
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:client_id",
440+
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/analysis:remote_analysis_cache_client",
440441
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec",
441442
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:serialization-constant",
442443
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/testutils",

0 commit comments

Comments
 (0)