Feat: Add Custom OpenTelemetry Exporter in for Service Metrics

package.json

@@ -55,13 +55,17 @@
  },
  "dependencies": {
  "@google-cloud/common": "^6.0.0",
+ "@google-cloud/monitoring": "^5.0.0",
+ "@google-cloud/opentelemetry-resource-util": "^2.4.0",
  "@google-cloud/precise-date": "^5.0.0",
  "@google-cloud/projectify": "^5.0.0",
  "@google-cloud/promisify": "^5.0.0",
+ "@grpc/grpc-js": "^1.13.2",
  "@grpc/proto-loader": "^0.7.13",
  "@opentelemetry/api": "^1.9.0",
  "@opentelemetry/context-async-hooks": "^2.0.0",
  "@opentelemetry/core": "^2.0.0",
+ "@opentelemetry/sdk-metrics": "^1.30.1",
  "@opentelemetry/semantic-conventions": "^1.30.0",
  "@types/big.js": "^6.2.2",
  "@types/stack-trace": "^0.0.33",

src/metrics/README.md

@@ -0,0 +1,19 @@
+# Custom Metric Exporter
+The custom metric exporter, as defined in [spanner-metrics-exporter.ts](./spanner-metrics-exporter.ts), is designed to work in conjunction with OpenTelemetry and the Spanner client. It converts data into its protobuf equivalent and sends it to Google Cloud Monitoring.
+
+## Filtering Criteria
+The exporter filters metrics based on the following conditions, utilizing values defined in [constants.ts](./constants.ts):
+
+* Metrics with a scope set to `spanner-nodejs`.
+* Metrics with one of the following predefined names:
+ * `attempt_latencies`
+ * `attempt_count`
+ * `operation_latencies`
+ * `operation_count`
+ * `gfe_latencies`
+ * `gfe_connectivity_error_count`
+
+## Service Endpoint
+The exporter sends metrics to the Google Cloud Monitoring [service endpoint](https://cloud.google.com/python/docs/reference/monitoring/latest/google.cloud.monitoring_v3.services.metric_service.MetricServiceClient#google_cloud_monitoring_v3_services_metric_service_MetricServiceClient_create_service_time_series), distinct from the regular client endpoint. This service endpoint operates under a different quota limit than the user endpoint and features an additional server-side filter that only permits a predefined set of metrics to pass through.
+
+When introducing new service metrics, it is essential to ensure they are allowed through by the server-side filter as well.

src/metrics/constants.ts

@@ -0,0 +1,62 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+export const SPANNER_METER_NAME = 'spanner-nodejs';
+export const CLIENT_METRICS_PREFIX = 'spanner.googleapis.com/internal/client';
+export const SPANNER_RESOURCE_TYPE = 'spanner_instance_client';
+
+// Monitored resource labels
+export const MONITORED_RES_LABEL_KEY_PROJECT = 'project_id';
+export const MONITORED_RES_LABEL_KEY_INSTANCE = 'instance_id';
+export const MONITORED_RES_LABEL_KEY_INSTANCE_CONFIG = 'instance_config';
+export const MONITORED_RES_LABEL_KEY_LOCATION = 'location';
+export const MONITORED_RES_LABEL_KEY_CLIENT_HASH = 'client_hash';
+export const MONITORED_RESOURCE_LABELS = new Set([
+ MONITORED_RES_LABEL_KEY_PROJECT,
+ MONITORED_RES_LABEL_KEY_INSTANCE,
+ MONITORED_RES_LABEL_KEY_INSTANCE_CONFIG,
+ MONITORED_RES_LABEL_KEY_LOCATION,
+ MONITORED_RES_LABEL_KEY_CLIENT_HASH,
+]);
+
+// Metric labels
+export const METRIC_LABEL_KEY_CLIENT_UID = 'client_uid';
+export const METRIC_LABEL_KEY_CLIENT_NAME = 'client_name';
+export const METRIC_LABEL_KEY_DATABASE = 'database';
+export const METRIC_LABEL_KEY_METHOD = 'method';
+export const METRIC_LABEL_KEY_STATUS = 'status';
+export const METRIC_LABELS = new Set([
+ METRIC_LABEL_KEY_CLIENT_UID,
+ METRIC_LABEL_KEY_CLIENT_NAME,
+ METRIC_LABEL_KEY_DATABASE,
+ METRIC_LABEL_KEY_METHOD,
+ METRIC_LABEL_KEY_STATUS,
+]);
+
+// Metric names
+export const METRIC_NAME_OPERATION_LATENCIES = 'operation_latencies';
+export const METRIC_NAME_ATTEMPT_LATENCIES = 'attempt_latencies';
+export const METRIC_NAME_OPERATION_COUNT = 'operation_count';
+export const METRIC_NAME_ATTEMPT_COUNT = 'attempt_count';
+export const METRIC_NAME_GFE_LATENCIES = 'gfe_latencies';
+export const METRIC_NAME_GFE_CONNECTIVITY_ERROR_COUNT =
+ 'gfe_connectivity_error_count';
+export const METRIC_NAMES = new Set([
+ METRIC_NAME_OPERATION_LATENCIES,
+ METRIC_NAME_ATTEMPT_LATENCIES,
+ METRIC_NAME_GFE_LATENCIES,
+ METRIC_NAME_OPERATION_COUNT,
+ METRIC_NAME_ATTEMPT_COUNT,
+ METRIC_NAME_GFE_CONNECTIVITY_ERROR_COUNT,
+]);

src/metrics/external-types.ts

@@ -0,0 +1,37 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {GoogleAuth} from 'google-auth-library';
+
+export interface ExporterOptions {
+ /**
+ * Optional authentication options for Google services.
+ */
+ auth: GoogleAuth;
+}
+
+export enum MetricKind {
+ UNSPECIFIED = 'METRIC_KIND_UNSPECIFIED',
+ GAUGE = 'GAUGE',
+ DELTA = 'DELTA',
+ CUMULATIVE = 'CUMULATIVE',
+}
+
+/** The value type of a metric. */
+export enum ValueType {
+ VALUE_TYPE_UNSPECIFIED = 'VALUE_TYPE_UNSPECIFIED',
+ INT64 = 'INT64',
+ DOUBLE = 'DOUBLE',
+ DISTRIBUTION = 'DISTRIBUTION',
+}

src/metrics/spanner-metrics-exporter.ts

@@ -0,0 +1,138 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {PushMetricExporter, ResourceMetrics} from '@opentelemetry/sdk-metrics';
+import {ExportResult, ExportResultCode} from '@opentelemetry/core';
+import {ExporterOptions} from './external-types';
+import {MetricServiceClient} from '@google-cloud/monitoring';
+import {transformResourceMetricToTimeSeriesArray} from './transform';
+import {status} from '@grpc/grpc-js';
+
+// Stackdriver Monitoring v3 only accepts up to 200 TimeSeries per
+// CreateTimeSeries call.
+export const MAX_BATCH_EXPORT_SIZE = 200;
+
+/**
+ * Format and sends metrics information to Google Cloud Monitoring.
+ */
+export class CloudMonitoringMetricsExporter implements PushMetricExporter {
+ private _projectId: string | void | Promise<string | void>;
+
+ private readonly _client: MetricServiceClient;
+
+ constructor({auth}: ExporterOptions) {
+ this._client = new MetricServiceClient({auth: auth});
+
+ // Start this async process as early as possible. It will be
+ // awaited on the first export because constructors are synchronous
+ this._projectId = auth.getProjectId().catch(err => {
+ console.error(err);
+ });
+ }
+
+ /**
+ * Implementation for {@link PushMetricExporter.export}.
+ * Calls the async wrapper method {@link _exportAsync} and
+ * assures no rejected promises bubble up to the caller.
+ *
+ * @param metrics Metrics to be sent to the Google Cloud Monitoring backend
+ * @param resultCallback result callback to be called on finish
+ */
+ export(
+ metrics: ResourceMetrics,
+ resultCallback: (result: ExportResult) => void,
+ ): void {
+ this._exportAsync(metrics).then(resultCallback, err => {
+ console.error(err.message);
+ resultCallback({code: ExportResultCode.FAILED, error: err});
+ });
+ }
+
+ async shutdown(): Promise<void> {}
+ async forceFlush(): Promise<void> {}
+
+ /**
+ * Asnyc wrapper for the {@link export} implementation.
+ * Writes the current values of all exported {@link MetricRecord}s
+ * to the Google Cloud Monitoring backend.
+ *
+ * @param resourceMetrics Metrics to be sent to the Google Cloud Monitoring backend
+ */
+ private async _exportAsync(
+ resourceMetrics: ResourceMetrics,
+ ): Promise<ExportResult> {
+ if (this._projectId instanceof Promise) {
+ this._projectId = await this._projectId;
+ }
+
+ if (!this._projectId) {
+ const error = new Error('expecting a non-blank ProjectID');
+ console.error(error.message);
+ return {code: ExportResultCode.FAILED, error};
+ }
+
+ const timeSeriesList =
+ transformResourceMetricToTimeSeriesArray(resourceMetrics);
+
+ let failure: {sendFailed: false} | {sendFailed: true; error: Error} = {
+ sendFailed: false,
+ };
+ await Promise.all(
+ this._partitionList(timeSeriesList, MAX_BATCH_EXPORT_SIZE).map(
+ async batchedTimeSeries => this._sendTimeSeries(batchedTimeSeries),
+ ),
+ ).catch(e => {
+ const error = e as {code: number};
+ if (error.code === status.PERMISSION_DENIED) {
+ console.warn(
+ `Need monitoring metric writer permission on project ${this._projectId}. Follow https://cloud.google.com/spanner/docs/view-manage-client-side-metrics#access-client-side-metrics to set up permissions`,
+ );
+ }
+ const err = asError(e);
+ err.message = `Send TimeSeries failed: ${err.message}`;
+ failure = {sendFailed: true, error: err};
+ console.error(`ERROR: ${err.message}`);
+ });
+
+ return failure.sendFailed
+ ? {
+ code: ExportResultCode.FAILED,
+ error: (failure as {sendFailed: boolean; error: Error}).error,
+ }
+ : {code: ExportResultCode.SUCCESS};
+ }
+
+ private async _sendTimeSeries(timeSeries) {
+ if (timeSeries.length === 0) {
+ return Promise.resolve();
+ }
+
+ // TODO: Use createServiceTimeSeries when it is available
+ await this._client.createTimeSeries({
+ name: `projects/${this._projectId}`,
+ timeSeries: timeSeries,
+ });
+ }
+
+ /** Returns the minimum number of arrays of max size chunkSize, partitioned from the given array. */
+ private _partitionList(list, chunkSize: number) {
+ return Array.from({length: Math.ceil(list.length / chunkSize)}, (_, i) =>
+ list.slice(i * chunkSize, (i + 1) * chunkSize),
+ );
+ }
+}
+
+function asError(error: unknown): Error {
+ return error instanceof Error ? error : new Error(String(error));
+}