chore: better error message when metric publish failed (#2214)

Also reduces logging when there are continuous failures (like permission denied). Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://togithub.com/googleapis/java-bigtable/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [ ] Appropriate docs were updated (if necessary) - [ ] Rollback plan is reviewed and LGTMed - [ ] All new data plane features have a completed end to end testing plan Fixes #<issue_number_goes_here> ☕️ If you write sample code, please follow the [samples format]( https://togithub.com/GoogleCloudPlatform/java-docs-samples/blob/main/SAMPLE_FORMAT.md).

Author: mutianf

README.md

@@ -19,7 +19,7 @@ If you are using Maven with [BOM][libraries-bom], add this to your pom.xml file:
  <dependency>
  <groupId>com.google.cloud</groupId>
  <artifactId>libraries-bom</artifactId>
- <version>26.37.0</version>
+ <version>26.25.0</version>
  <type>pom</type>
  <scope>import</scope>
  </dependency>

google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/metrics/BigtableCloudMonitoringExporter.java

@@ -34,6 +34,7 @@
 import com.google.api.gax.core.CredentialsProvider;
 import com.google.api.gax.core.FixedCredentialsProvider;
 import com.google.api.gax.core.NoCredentialsProvider;
+import com.google.api.gax.rpc.PermissionDeniedException;
 import com.google.auth.Credentials;
 import com.google.cloud.monitoring.v3.MetricServiceClient;
 import com.google.cloud.monitoring.v3.MetricServiceSettings;
@@ -96,6 +97,9 @@ public final class BigtableCloudMonitoringExporter implements MetricExporter {
 
  private CompletableResultCode lastExportCode;
 
+ private final AtomicBoolean bigtableExportFailureLogged = new AtomicBoolean(false);
+ private final AtomicBoolean applicationExportFailureLogged = new AtomicBoolean(false);
+
  private static final ImmutableList<String> BIGTABLE_TABLE_METRICS =
  ImmutableSet.of(
  OPERATION_LATENCIES_NAME,
@@ -227,15 +231,24 @@ private CompletableResultCode exportBigtableResourceMetrics(Collection<MetricDat
  new ApiFutureCallback<Empty>() {
  @Override
  public void onFailure(Throwable throwable) {
- logger.log(
- Level.WARNING,
- "createServiceTimeSeries request failed for bigtable metrics. ",
- throwable);
+ if (bigtableExportFailureLogged.compareAndSet(false, true)) {
+ String msg = "createServiceTimeSeries request failed for bigtable metrics.";
+ if (throwable instanceof PermissionDeniedException) {
+ msg +=
+ String.format(
+ " Need monitoring metric writer permission on project=%s. Follow https://cloud.google.com/bigtable/docs/client-side-metrics-setup to set up permissions.",
+ projectName.getProject());
+ }
+ logger.log(Level.WARNING, msg, throwable);
+ }
  bigtableExportCode.fail();
  }
 
  @Override
  public void onSuccess(Empty empty) {
+ // When an export succeeded reset the export failure flag to false so if there's a
+ // transient failure it'll be logged.
+ bigtableExportFailureLogged.set(false);
  bigtableExportCode.succeed();
  }
  },
@@ -295,15 +308,24 @@ private CompletableResultCode exportApplicationResourceMetrics(
  new ApiFutureCallback<Empty>() {
  @Override
  public void onFailure(Throwable throwable) {
- logger.log(
- Level.WARNING,
- "createServiceTimeSeries request failed for per connection error metrics.",
- throwable);
+ if (applicationExportFailureLogged.compareAndSet(false, true)) {
+ String msg = "createServiceTimeSeries request failed for bigtable metrics.";
+ if (throwable instanceof PermissionDeniedException) {
+ msg +=
+ String.format(
+ " Need monitoring metric writer permission on project=%s. Follow https://cloud.google.com/bigtable/docs/client-side-metrics-setup to set up permissions.",
+ projectName.getProject());
+ }
+ logger.log(Level.WARNING, msg, throwable);
+ }
  exportCode.fail();
  }
 
  @Override
  public void onSuccess(Empty empty) {
+ // When an export succeeded reset the export failure flag to false so if there's a
+ // transient failure it'll be logged.
+ applicationExportFailureLogged.set(false);
  exportCode.succeed();
  }
  },