Skip to content

Commit de82f78

Browse files
KiranmayiBKiranmayiB
authored
feat: expose GFE latency metrics (#1473)
* GFE Latency * feat: GFE Latency * Busy waiting for header missing count * Thread sleep before checking view Data * lint errors and changing method signatures in unit test * lint changes * Adding pattern for Admin calls * Correcting regex pattern
1 parent ff2a713 commit de82f78

File tree

5 files changed

+675
-2
lines changed

5 files changed

+675
-2
lines changed

‎google-cloud-spanner/pom.xml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@
9898
<groupId>org.apache.maven.plugins</groupId>
9999
<artifactId>maven-dependency-plugin</artifactId>
100100
<configuration>
101-
<ignoredDependencies>io.grpc:grpc-protobuf-lite,org.hamcrest:hamcrest,org.hamcrest:hamcrest-core,com.google.errorprone:error_prone_annotations,org.openjdk.jmh:jmh-generator-annprocess,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,javax.annotation:javax.annotation-api</ignoredDependencies>
101+
<ignoredDependencies>io.grpc:grpc-protobuf-lite,org.hamcrest:hamcrest,org.hamcrest:hamcrest-core,com.google.errorprone:error_prone_annotations,org.openjdk.jmh:jmh-generator-annprocess,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,javax.annotation:javax.annotation-api,io.opencensus:opencensus-impl</ignoredDependencies>
102102
</configuration>
103103
</plugin>
104104
</plugins>
@@ -179,6 +179,11 @@
179179
<groupId>io.opencensus</groupId>
180180
<artifactId>opencensus-contrib-grpc-util</artifactId>
181181
</dependency>
182+
<dependency>
183+
<groupId>io.opencensus</groupId>
184+
<artifactId>opencensus-impl</artifactId>
185+
<scope>test</scope>
186+
</dependency>
182187
<dependency>
183188
<groupId>com.google.auth</groupId>
184189
<artifactId>google-auth-library-oauth2-http</artifactId>

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
/*
2+
* Copyright 2021 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.spanner.spi.v1;
17+
18+
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.DATABASE_ID;
19+
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.INSTANCE_ID;
20+
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.METHOD;
21+
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.PROJECT_ID;
22+
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.SPANNER_GFE_HEADER_MISSING_COUNT;
23+
import static com.google.cloud.spanner.spi.v1.SpannerRpcViews.SPANNER_GFE_LATENCY;
24+
25+
import io.grpc.CallOptions;
26+
import io.grpc.Channel;
27+
import io.grpc.ClientCall;
28+
import io.grpc.ClientInterceptor;
29+
import io.grpc.ForwardingClientCall.SimpleForwardingClientCall;
30+
import io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener;
31+
import io.grpc.Metadata;
32+
import io.grpc.MethodDescriptor;
33+
import io.opencensus.stats.MeasureMap;
34+
import io.opencensus.stats.Stats;
35+
import io.opencensus.stats.StatsRecorder;
36+
import io.opencensus.tags.TagContext;
37+
import io.opencensus.tags.TagValue;
38+
import io.opencensus.tags.Tagger;
39+
import io.opencensus.tags.Tags;
40+
import java.util.logging.Level;
41+
import java.util.logging.Logger;
42+
import java.util.regex.Matcher;
43+
import java.util.regex.Pattern;
44+
45+
/**
46+
* Intercepts all gRPC calls to extract server-timing header. Captures GFE Latency and GFE Header
47+
* Missing count metrics.
48+
*/
49+
class HeaderInterceptor implements ClientInterceptor {
50+
51+
private static final Metadata.Key<String> SERVER_TIMING_HEADER_KEY =
52+
Metadata.Key.of("server-timing", Metadata.ASCII_STRING_MARSHALLER);
53+
private static final Pattern SERVER_TIMING_HEADER_PATTERN = Pattern.compile(".*dur=(?<dur>\\d+)");
54+
private static final Metadata.Key<String> GOOGLE_CLOUD_RESOURCE_PREFIX_KEY =
55+
Metadata.Key.of("google-cloud-resource-prefix", Metadata.ASCII_STRING_MARSHALLER);
56+
private static final Pattern GOOGLE_CLOUD_RESOURCE_PREFIX_PATTERN =
57+
Pattern.compile(
58+
".*projects/(?<project>\\p{ASCII}[^/]*)(/instances/(?<instance>\\p{ASCII}[^/]*))?(/databases/(?<database>\\p{ASCII}[^/]*))?");
59+
60+
// Get the global singleton Tagger object.
61+
private static final Tagger TAGGER = Tags.getTagger();
62+
private static final StatsRecorder STATS_RECORDER = Stats.getStatsRecorder();
63+
64+
private static final Logger LOGGER = Logger.getLogger(HeaderInterceptor.class.getName());
65+
private static final Level LEVEL = Level.INFO;
66+
67+
HeaderInterceptor() {}
68+
69+
@Override
70+
public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(
71+
MethodDescriptor<ReqT, RespT> method, CallOptions callOptions, Channel next) {
72+
return new SimpleForwardingClientCall<ReqT, RespT>(next.newCall(method, callOptions)) {
73+
@Override
74+
public void start(Listener<RespT> responseListener, Metadata headers) {
75+
TagContext tagContext = getTagContext(headers, method.getFullMethodName());
76+
super.start(
77+
new SimpleForwardingClientCallListener<RespT>(responseListener) {
78+
@Override
79+
public void onHeaders(Metadata metadata) {
80+
81+
processHeader(metadata, tagContext);
82+
super.onHeaders(metadata);
83+
}
84+
},
85+
headers);
86+
}
87+
};
88+
}
89+
90+
private void processHeader(Metadata metadata, TagContext tagContext) {
91+
MeasureMap measureMap = STATS_RECORDER.newMeasureMap();
92+
if (metadata.get(SERVER_TIMING_HEADER_KEY) != null) {
93+
String serverTiming = metadata.get(SERVER_TIMING_HEADER_KEY);
94+
Matcher matcher = SERVER_TIMING_HEADER_PATTERN.matcher(serverTiming);
95+
if (matcher.find()) {
96+
try {
97+
long latency = Long.parseLong(matcher.group("dur"));
98+
measureMap.put(SPANNER_GFE_LATENCY, latency).record(tagContext);
99+
measureMap.put(SPANNER_GFE_HEADER_MISSING_COUNT, 0L).record(tagContext);
100+
} catch (NumberFormatException e) {
101+
LOGGER.log(LEVEL, "Invalid server-timing object in header", matcher.group("dur"));
102+
}
103+
}
104+
} else {
105+
measureMap.put(SPANNER_GFE_HEADER_MISSING_COUNT, 1L).record(tagContext);
106+
}
107+
}
108+
109+
private TagContext getTagContext(
110+
String method, String projectId, String instanceId, String databaseId) {
111+
return TAGGER
112+
.currentBuilder()
113+
.putLocal(PROJECT_ID, TagValue.create(projectId))
114+
.putLocal(INSTANCE_ID, TagValue.create(instanceId))
115+
.putLocal(DATABASE_ID, TagValue.create(databaseId))
116+
.putLocal(METHOD, TagValue.create(method))
117+
.build();
118+
}
119+
120+
private TagContext getTagContext(Metadata headers, String method) {
121+
String projectId = "undefined-project";
122+
String instanceId = "undefined-database";
123+
String databaseId = "undefined-database";
124+
if (headers.get(GOOGLE_CLOUD_RESOURCE_PREFIX_KEY) != null) {
125+
String googleResourcePrefix = headers.get(GOOGLE_CLOUD_RESOURCE_PREFIX_KEY);
126+
Matcher matcher = GOOGLE_CLOUD_RESOURCE_PREFIX_PATTERN.matcher(googleResourcePrefix);
127+
if (matcher.find()) {
128+
projectId = matcher.group("project");
129+
if (matcher.group("instance") != null) {
130+
instanceId = matcher.group("instance");
131+
}
132+
if (matcher.group("database") != null) {
133+
databaseId = matcher.group("database");
134+
}
135+
} else {
136+
LOGGER.log(LEVEL, "Error parsing google cloud resource header: " + googleResourcePrefix);
137+
}
138+
}
139+
return getTagContext(method, projectId, instanceId, databaseId);
140+
}
141+
}

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerInterceptorProvider.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ public class SpannerInterceptorProvider implements GrpcInterceptorProvider {
3333
private static final List<ClientInterceptor> defaultInterceptors =
3434
ImmutableList.of(
3535
new SpannerErrorInterceptor(),
36-
new LoggingInterceptor(Logger.getLogger(GapicSpannerRpc.class.getName()), Level.FINER));
36+
new LoggingInterceptor(Logger.getLogger(GapicSpannerRpc.class.getName()), Level.FINER),
37+
new HeaderInterceptor());
3738

3839
private final List<ClientInterceptor> clientInterceptors;
3940

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/SpannerRpcViews.java‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/*
2+
* Copyright 2021 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.spanner.spi.v1;
17+
18+
import com.google.common.annotations.VisibleForTesting;
19+
import com.google.common.collect.ImmutableList;
20+
import io.opencensus.stats.Aggregation;
21+
import io.opencensus.stats.Aggregation.Distribution;
22+
import io.opencensus.stats.Aggregation.Sum;
23+
import io.opencensus.stats.BucketBoundaries;
24+
import io.opencensus.stats.Measure.MeasureLong;
25+
import io.opencensus.stats.Stats;
26+
import io.opencensus.stats.View;
27+
import io.opencensus.stats.ViewManager;
28+
import io.opencensus.tags.TagKey;
29+
import java.util.Arrays;
30+
import java.util.Collections;
31+
import java.util.List;
32+
33+
@VisibleForTesting
34+
public class SpannerRpcViews {
35+
36+
/** Unit to represent milliseconds. */
37+
private static final String MILLISECOND = "ms";
38+
/** Unit to represent counts. */
39+
private static final String COUNT = "1";
40+
41+
/** TagKeys */
42+
public static final TagKey METHOD = TagKey.create("method");
43+
44+
public static final TagKey PROJECT_ID = TagKey.create("project_id");
45+
public static final TagKey INSTANCE_ID = TagKey.create("instance_id");
46+
public static final TagKey DATABASE_ID = TagKey.create("database");
47+
48+
/** GFE t4t7 latency extracted from server-timing header. */
49+
public static final MeasureLong SPANNER_GFE_LATENCY =
50+
MeasureLong.create(
51+
"cloud.google.com/java/spanner/gfe_latency",
52+
"Latency between Google's network receiving an RPC and reading back the first byte of the response",
53+
MILLISECOND);
54+
/** Number of responses without the server-timing header. */
55+
public static final MeasureLong SPANNER_GFE_HEADER_MISSING_COUNT =
56+
MeasureLong.create(
57+
"cloud.google.com/java/spanner/gfe_header_missing_count",
58+
"Number of RPC responses received without the server-timing header, most likely means that the RPC never reached Google's network",
59+
COUNT);
60+
61+
static final List<Double> RPC_MILLIS_BUCKET_BOUNDARIES =
62+
Collections.unmodifiableList(
63+
Arrays.asList(
64+
0.0, 0.01, 0.05, 0.1, 0.3, 0.6, 0.8, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 8.0, 10.0, 13.0,
65+
16.0, 20.0, 25.0, 30.0, 40.0, 50.0, 65.0, 80.0, 100.0, 130.0, 160.0, 200.0, 250.0,
66+
300.0, 400.0, 500.0, 650.0, 800.0, 1000.0, 2000.0, 5000.0, 10000.0, 20000.0, 50000.0,
67+
100000.0));
68+
static final Aggregation AGGREGATION_WITH_MILLIS_HISTOGRAM =
69+
Distribution.create(BucketBoundaries.create(RPC_MILLIS_BUCKET_BOUNDARIES));
70+
static final View SPANNER_GFE_LATENCY_VIEW =
71+
View.create(
72+
View.Name.create("cloud.google.com/java/spanner/gfe_latency"),
73+
"Latency between Google's network receiving an RPC and reading back the first byte of the response",
74+
SPANNER_GFE_LATENCY,
75+
AGGREGATION_WITH_MILLIS_HISTOGRAM,
76+
ImmutableList.of(METHOD, PROJECT_ID, INSTANCE_ID, DATABASE_ID));
77+
78+
private static final Aggregation SUM = Sum.create();
79+
static final View SPANNER_GFE_HEADER_MISSING_COUNT_VIEW =
80+
View.create(
81+
View.Name.create("cloud.google.com/java/spanner/gfe_header_missing_count"),
82+
"Number of RPC responses received without the server-timing header, most likely means that the RPC never reached Google's network",
83+
SPANNER_GFE_HEADER_MISSING_COUNT,
84+
SUM,
85+
ImmutableList.of(METHOD, PROJECT_ID, INSTANCE_ID, DATABASE_ID));
86+
87+
public static ViewManager viewManager = Stats.getViewManager();
88+
89+
/**
90+
* Register views for GFE metrics, including gfe_latency and gfe_header_missing_count. gfe_latency
91+
* measures the latency between Google's network receives an RPC and reads back the first byte of
92+
* the response. gfe_header_missing_count is a counter of the number of RPC responses without a
93+
* server-timing header.
94+
*/
95+
@VisibleForTesting
96+
public static void registerGfeLatencyAndHeaderMissingCountViews() {
97+
viewManager.registerView(SPANNER_GFE_LATENCY_VIEW);
98+
viewManager.registerView(SPANNER_GFE_HEADER_MISSING_COUNT_VIEW);
99+
}
100+
101+
/**
102+
* Register GFE Latency view. gfe_latency measures the latency between Google's network receives
103+
* an RPC and reads back the first byte of the response.
104+
*/
105+
@VisibleForTesting
106+
public static void registerGfeLatencyView() {
107+
viewManager.registerView(SPANNER_GFE_LATENCY_VIEW);
108+
}
109+
110+
/**
111+
* Register GFE Header Missing Count view. gfe_header_missing_count is a counter of the number of
112+
* RPC responses without a server-timing header.
113+
*/
114+
@VisibleForTesting
115+
public static void registerGfeHeaderMissingCountView() {
116+
viewManager.registerView(SPANNER_GFE_HEADER_MISSING_COUNT_VIEW);
117+
}
118+
}

0 commit comments

Comments
 (0)