Skip to content

Commit 0e91197

Browse files
mutianfmutianf
committed
feat: add built in metrics measure and views
1 parent cf8b38b commit 0e91197

File tree

8 files changed

+1110
-28
lines changed

8 files changed

+1110
-28
lines changed

‎google-cloud-bigtable-stats/pom.xml‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,15 @@
2929
</dependencyManagement>
3030

3131
<dependencies>
32+
<dependency>
33+
<groupId>com.google.api</groupId>
34+
<artifactId>gax</artifactId>
35+
</dependency>
36+
<dependency>
37+
<groupId>com.google.api</groupId>
38+
<artifactId>api-common</artifactId>
39+
</dependency>
40+
3241
<dependency>
3342
<groupId>io.opencensus</groupId>
3443
<artifactId>opencensus-api</artifactId>
@@ -38,8 +47,30 @@
3847
<artifactId>opencensus-impl</artifactId>
3948
</dependency>
4049
<dependency>
41-
<groupId>io.opencensus</groupId>
42-
<artifactId>opencensus-exporter-stats-stackdriver</artifactId>
50+
<groupId>com.google.guava</groupId>
51+
<artifactId>guava</artifactId>
52+
</dependency>
53+
54+
55+
<dependency>
56+
<groupId>com.google.truth.extensions</groupId>
57+
<artifactId>truth-proto-extension</artifactId>
58+
<scope>test</scope>
59+
</dependency>
60+
<dependency>
61+
<groupId>com.google.truth</groupId>
62+
<artifactId>truth</artifactId>
63+
<scope>test</scope>
64+
</dependency>
65+
<dependency>
66+
<groupId>junit</groupId>
67+
<artifactId>junit</artifactId>
68+
<scope>test</scope>
69+
</dependency>
70+
<dependency>
71+
<groupId>io.grpc</groupId>
72+
<artifactId>grpc-api</artifactId>
73+
<scope>test</scope>
4374
</dependency>
4475
</dependencies>
4576

‎google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMeasureConstants.java‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.stats;
17+
18+
import static io.opencensus.stats.Measure.MeasureLong;
19+
20+
import com.google.api.core.InternalApi;
21+
import io.opencensus.tags.TagKey;
22+
23+
@InternalApi("For internal use only")
24+
public class BuiltinMeasureConstants {
25+
// TagKeys
26+
public static final TagKey PROJECT_ID = TagKey.create("project_id");
27+
public static final TagKey INSTANCE_ID = TagKey.create("instance_id");
28+
public static final TagKey APP_PROFILE = TagKey.create("app_profile");
29+
public static final TagKey METHOD = TagKey.create("method");
30+
public static final TagKey STREAMING = TagKey.create("streaming");
31+
public static final TagKey STATUS = TagKey.create("status");
32+
public static final TagKey CLIENT_NAME = TagKey.create("client_name");
33+
public static final TagKey CLIENT_ID = TagKey.create("client_id");
34+
35+
// Monitored resource TagKeys
36+
public static final TagKey TABLE = TagKey.create("table");
37+
public static final TagKey CLUSTER = TagKey.create("cluster");
38+
public static final TagKey ZONE = TagKey.create("zone");
39+
40+
// Units
41+
private static final String COUNT = "1";
42+
private static final String MILLISECOND = "ms";
43+
44+
// Measurements
45+
static final MeasureLong OPERATION_LATENCIES =
46+
MeasureLong.create(
47+
"bigtable.googleapis.com/internal/client/operation_latencies",
48+
"Total time until final operation success or failure, including retries and backoff.",
49+
MILLISECOND);
50+
51+
static final MeasureLong ATTEMPT_LATENCIES =
52+
MeasureLong.create(
53+
"bigtable.googleapis.com/internal/client/attempt_latencies",
54+
"Client observed latency per RPC attempt.",
55+
MILLISECOND);
56+
57+
static final MeasureLong RETRY_COUNT =
58+
MeasureLong.create(
59+
"bigtable.googleapis.com/internal/client/retry_count",
60+
"The number of additional RPCs sent after the initial attempt.",
61+
COUNT);
62+
63+
static final MeasureLong FIRST_RESPONSE_LATENCIES =
64+
MeasureLong.create(
65+
"bigtable.googleapis.com/internal/client/first_response_latencies",
66+
"Latency from operation start until the response headers were received. The publishing of the measurement will be delayed until the attempt response has been received.",
67+
MILLISECOND);
68+
69+
static final MeasureLong SERVER_LATENCIES =
70+
MeasureLong.create(
71+
"bigtable.googleapis.com/internal/client/server_latencies",
72+
"The latency measured from the moment that the RPC entered the Google data center until the RPC was completed.",
73+
MILLISECOND);
74+
75+
static final MeasureLong CONNECTIVITY_ERROR_COUNT =
76+
MeasureLong.create(
77+
"bigtable.googleapis.com/internal/client/connectivity_error_count",
78+
"Number of requests that failed to reach the Google datacenter. (Requests without google response headers).",
79+
COUNT);
80+
81+
static final MeasureLong APPLICATION_LATENCIES =
82+
MeasureLong.create(
83+
"bigtable.googleapis.com/internal/client/application_latencies",
84+
"The latency of the client application consuming available response data.",
85+
MILLISECOND);
86+
87+
static final MeasureLong THROTTLING_LATENCIES =
88+
MeasureLong.create(
89+
"bigtable.googleapis.com/internal/client/throttling_latencies",
90+
"The artificial latency introduced by the client to limit the number of outstanding requests. The publishing of the measurement will be delayed until the attempt trailers have been received.",
91+
MILLISECOND);
92+
}

‎google-cloud-bigtable-stats/src/main/java/com/google/cloud/bigtable/stats/BuiltinMetricsRecorder.java‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
/*
2+
* Copyright 2022 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
package com.google.cloud.bigtable.stats;
17+
18+
import com.google.api.core.InternalApi;
19+
import com.google.api.gax.tracing.ApiTracerFactory.OperationType;
20+
import com.google.api.gax.tracing.SpanName;
21+
import io.opencensus.stats.MeasureMap;
22+
import io.opencensus.stats.StatsRecorder;
23+
import io.opencensus.tags.TagContext;
24+
import io.opencensus.tags.TagContextBuilder;
25+
import io.opencensus.tags.TagKey;
26+
import io.opencensus.tags.TagValue;
27+
import io.opencensus.tags.Tagger;
28+
import io.opencensus.tags.Tags;
29+
import java.util.Map;
30+
31+
/** Add built-in metrics to the measure map * */
32+
@InternalApi("For internal use only")
33+
public class BuiltinMetricsRecorder {
34+
35+
private final OperationType operationType;
36+
37+
private final Tagger tagger;
38+
private final StatsRecorder statsRecorder;
39+
private final TagContext parentContext;
40+
private final SpanName spanName;
41+
private final Map<String, String> statsAttributes;
42+
43+
private MeasureMap attemptLevelNoStreaming;
44+
private MeasureMap attemptLevelWithStreaming;
45+
private MeasureMap operationLevelNoStreaming;
46+
private MeasureMap operationLevelWithStreaming;
47+
48+
public BuiltinMetricsRecorder(
49+
OperationType operationType,
50+
SpanName spanName,
51+
Map<String, String> statsAttributes,
52+
StatsWrapper builtinMetricsWrapper) {
53+
this.operationType = operationType;
54+
this.tagger = Tags.getTagger();
55+
this.statsRecorder = builtinMetricsWrapper.getStatsRecorder();
56+
this.spanName = spanName;
57+
this.parentContext = tagger.getCurrentTagContext();
58+
this.statsAttributes = statsAttributes;
59+
60+
this.attemptLevelNoStreaming = statsRecorder.newMeasureMap();
61+
this.attemptLevelWithStreaming = statsRecorder.newMeasureMap();
62+
this.operationLevelNoStreaming = statsRecorder.newMeasureMap();
63+
this.operationLevelWithStreaming = statsRecorder.newMeasureMap();
64+
}
65+
66+
public void recordAttemptLevelWithoutStreaming(
67+
String status, String tableId, String zone, String cluster) {
68+
TagContextBuilder tagCtx =
69+
newTagContextBuilder(tableId, zone, cluster)
70+
.putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status));
71+
72+
attemptLevelNoStreaming.record(tagCtx.build());
73+
}
74+
75+
public void recordAttemptLevelWithStreaming(
76+
String status, String tableId, String zone, String cluster) {
77+
TagContextBuilder tagCtx =
78+
newTagContextBuilder(tableId, zone, cluster)
79+
.putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status));
80+
81+
if (operationType == OperationType.ServerStreaming
82+
&& spanName.getMethodName().equals("ReadRows")) {
83+
tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true"));
84+
} else {
85+
tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false"));
86+
}
87+
88+
attemptLevelWithStreaming.record(tagCtx.build());
89+
}
90+
91+
public void recordOperationLevelWithoutStreaming(
92+
String status, String tableId, String zone, String cluster) {
93+
TagContextBuilder tagCtx =
94+
newTagContextBuilder(tableId, zone, cluster)
95+
.putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status));
96+
97+
operationLevelNoStreaming.record(tagCtx.build());
98+
}
99+
100+
public void recordOperationLevelWithStreaming(
101+
String status, String tableId, String zone, String cluster) {
102+
TagContextBuilder tagCtx =
103+
newTagContextBuilder(tableId, zone, cluster)
104+
.putLocal(BuiltinMeasureConstants.STATUS, TagValue.create(status));
105+
106+
if (operationType == OperationType.ServerStreaming
107+
&& spanName.getMethodName().equals("ReadRows")) {
108+
tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("true"));
109+
} else {
110+
tagCtx.putLocal(BuiltinMeasureConstants.STREAMING, TagValue.create("false"));
111+
}
112+
113+
operationLevelWithStreaming.record(tagCtx.build());
114+
}
115+
116+
public void recordOperationLatencies(long operationLatency) {
117+
operationLevelWithStreaming.put(BuiltinMeasureConstants.OPERATION_LATENCIES, operationLatency);
118+
}
119+
120+
public void recordAttemptLatency(long attemptLatency) {
121+
attemptLevelWithStreaming.put(BuiltinMeasureConstants.ATTEMPT_LATENCIES, attemptLatency);
122+
}
123+
124+
public void recordRetryCount(int attemptCount) {
125+
operationLevelNoStreaming.put(BuiltinMeasureConstants.RETRY_COUNT, attemptCount);
126+
}
127+
128+
public void recordApplicationLatency(long applicationLatency) {
129+
operationLevelWithStreaming.put(
130+
BuiltinMeasureConstants.APPLICATION_LATENCIES, applicationLatency);
131+
}
132+
133+
public void recordFirstResponseLatency(long firstResponseLatency) {
134+
operationLevelNoStreaming.put(
135+
BuiltinMeasureConstants.FIRST_RESPONSE_LATENCIES, firstResponseLatency);
136+
}
137+
138+
public void recordGfeLatencies(long serverLatency) {
139+
attemptLevelWithStreaming.put(BuiltinMeasureConstants.SERVER_LATENCIES, serverLatency);
140+
}
141+
142+
public void recordGfeMissingHeaders(long connectivityErrors) {
143+
attemptLevelNoStreaming.put(
144+
BuiltinMeasureConstants.CONNECTIVITY_ERROR_COUNT, connectivityErrors);
145+
}
146+
147+
public void recordBatchRequestThrottled(
148+
long throttledTimeMs, String tableId, String zone, String cluster) {
149+
MeasureMap measures =
150+
statsRecorder
151+
.newMeasureMap()
152+
.put(BuiltinMeasureConstants.THROTTLING_LATENCIES, throttledTimeMs);
153+
measures.record(newTagContextBuilder(tableId, zone, cluster).build());
154+
}
155+
156+
private TagContextBuilder newTagContextBuilder(String tableId, String zone, String cluster) {
157+
TagContextBuilder tagContextBuilder =
158+
tagger
159+
.toBuilder(parentContext)
160+
.putLocal(BuiltinMeasureConstants.CLIENT_NAME, TagValue.create("bigtable-java"))
161+
.putLocal(BuiltinMeasureConstants.METHOD, TagValue.create(spanName.toString()))
162+
.putLocal(BuiltinMeasureConstants.TABLE, TagValue.create(tableId))
163+
.putLocal(BuiltinMeasureConstants.ZONE, TagValue.create(zone))
164+
.putLocal(BuiltinMeasureConstants.CLUSTER, TagValue.create(cluster));
165+
for (Map.Entry<String, String> entry : statsAttributes.entrySet()) {
166+
tagContextBuilder.putLocal(TagKey.create(entry.getKey()), TagValue.create(entry.getValue()));
167+
}
168+
return tagContextBuilder;
169+
}
170+
}

0 commit comments

Comments
 (0)