fix: stop sending RPCs on InstanceNotFound (#61)

The session pool should handle InstanceNotFound in the same way as DatabaseNotFound errors, i.e. stop sending RPCs and also consider an instance that has been re-created with the same name as a different instance, and hence require the user to re-create any database client before it can be used again. Fixes #60

Author: olavloite

google-cloud-spanner/src/main/java/com/google/cloud/spanner/InstanceNotFoundException.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright 2020 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.cloud.spanner;
+
+import com.google.cloud.spanner.SpannerException.ResourceNotFoundException;
+import com.google.rpc.ResourceInfo;
+import javax.annotation.Nullable;
+
+/**
+ * Exception thrown by Cloud Spanner when an operation detects that the instance that is being used
+ * no longer exists. This type of error has its own subclass as it is a condition that should cause
+ * the client library to stop trying to send RPCs to the backend until the user has taken action.
+ */
+public class InstanceNotFoundException extends ResourceNotFoundException {
+ private static final long serialVersionUID = 45297002L;
+
+ /** Private constructor. Use {@link SpannerExceptionFactory} to create instances. */
+ InstanceNotFoundException(
+ DoNotConstructDirectly token,
+ @Nullable String message,
+ ResourceInfo resourceInfo,
+ @Nullable Throwable cause) {
+ super(token, message, resourceInfo, cause);
+ }
+}

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SessionPool.java

@@ -26,6 +26,7 @@
 import com.google.cloud.spanner.Options.QueryOption;
 import com.google.cloud.spanner.Options.ReadOption;
 import com.google.cloud.spanner.SessionClient.SessionConsumer;
+import com.google.cloud.spanner.SpannerException.ResourceNotFoundException;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Function;
 import com.google.common.base.MoreObjects;
@@ -781,13 +782,14 @@ public void close() {
  if (lastException != null && isSessionNotFound(lastException)) {
  invalidateSession(this);
  } else {
- if (lastException != null && isDatabaseNotFound(lastException)) {
+ if (lastException != null && isDatabaseOrInstanceNotFound(lastException)) {
  // Mark this session pool as no longer valid and then release the session into the pool as
  // there is nothing we can do with it anyways.
  synchronized (lock) {
- SessionPool.this.databaseNotFound =
+ SessionPool.this.resourceNotFoundException =
  MoreObjects.firstNonNull(
- SessionPool.this.databaseNotFound, (DatabaseNotFoundException) lastException);
+ SessionPool.this.resourceNotFoundException,
+ (ResourceNotFoundException) lastException);
  }
  }
  lastException = null;
@@ -1075,7 +1077,7 @@ private static enum Position {
  private SettableFuture<Void> closureFuture;
 
  @GuardedBy("lock")
- private DatabaseNotFoundException databaseNotFound;
+ private ResourceNotFoundException resourceNotFoundException;
 
  @GuardedBy("lock")
  private final LinkedList<PooledSession> readSessions = new LinkedList<>();
@@ -1213,8 +1215,8 @@ private boolean isSessionNotFound(SpannerException e) {
  return e.getErrorCode() == ErrorCode.NOT_FOUND && e.getMessage().contains("Session not found");
  }
 
- private boolean isDatabaseNotFound(SpannerException e) {
- return e instanceof DatabaseNotFoundException;
+ private boolean isDatabaseOrInstanceNotFound(SpannerException e) {
+ return e instanceof DatabaseNotFoundException || e instanceof InstanceNotFoundException;
  }
 
  private boolean isPermissionDenied(SpannerException e) {
@@ -1249,7 +1251,7 @@ private PooledSession findSessionToKeepAlive(
  /** @return true if this {@link SessionPool} is still valid. */
  boolean isValid() {
  synchronized (lock) {
- return closureFuture == null && databaseNotFound == null;
+ return closureFuture == null && resourceNotFoundException == null;
  }
  }
 
@@ -1279,14 +1281,14 @@ PooledSession getReadSession() throws SpannerException {
  span.addAnnotation("Pool has been closed");
  throw new IllegalStateException("Pool has been closed");
  }
- if (databaseNotFound != null) {
+ if (resourceNotFoundException != null) {
  span.addAnnotation("Database has been deleted");
  throw SpannerExceptionFactory.newSpannerException(
  ErrorCode.NOT_FOUND,
  String.format(
  "The session pool has been invalidated because a previous RPC returned 'Database not found': %s",
- databaseNotFound.getMessage()),
- databaseNotFound);
+ resourceNotFoundException.getMessage()),
+ resourceNotFoundException);
  }
  sess = readSessions.poll();
  if (sess == null) {
@@ -1344,14 +1346,14 @@ PooledSession getReadWriteSession() {
  span.addAnnotation("Pool has been closed");
  throw new IllegalStateException("Pool has been closed");
  }
- if (databaseNotFound != null) {
+ if (resourceNotFoundException != null) {
  span.addAnnotation("Database has been deleted");
  throw SpannerExceptionFactory.newSpannerException(
  ErrorCode.NOT_FOUND,
  String.format(
  "The session pool has been invalidated because a previous RPC returned 'Database not found': %s",
- databaseNotFound.getMessage()),
- databaseNotFound);
+ resourceNotFoundException.getMessage()),
+ resourceNotFoundException);
  }
  sess = writePreparedSessions.poll();
  if (sess == null) {
@@ -1495,17 +1497,18 @@ private void handleCreateSessionsFailure(SpannerException e, int count) {
  break;
  }
  }
- this.databaseNotFound =
+ this.resourceNotFoundException =
  MoreObjects.firstNonNull(
- this.databaseNotFound, isDatabaseNotFound(e) ? (DatabaseNotFoundException) e : null);
+ this.resourceNotFoundException,
+ isDatabaseOrInstanceNotFound(e) ? (ResourceNotFoundException) e : null);
  }
  }
 
  private void handlePrepareSessionFailure(SpannerException e, PooledSession session) {
  synchronized (lock) {
  if (isSessionNotFound(e)) {
  invalidateSession(session);
- } else if (isDatabaseNotFound(e) || isPermissionDenied(e)) {
+ } else if (isDatabaseOrInstanceNotFound(e) || isPermissionDenied(e)) {
  // Database has been deleted or the user has no permission to write to this database. We
  // should stop trying to prepare any transactions. Also propagate the error to all waiters,
  // as any further waiting is pointless.
@@ -1520,10 +1523,10 @@ private void handlePrepareSessionFailure(SpannerException e, PooledSession sessi
  if (isClosed()) {
  decrementPendingClosures(1);
  }
- this.databaseNotFound =
+ this.resourceNotFoundException =
  MoreObjects.firstNonNull(
- this.databaseNotFound,
- isDatabaseNotFound(e) ? (DatabaseNotFoundException) e : null);
+ this.resourceNotFoundException,
+ isDatabaseOrInstanceNotFound(e) ? (ResourceNotFoundException) e : null);
  } else if (readWriteWaiters.size() > 0) {
  releaseSession(session, Position.FIRST);
  readWriteWaiters.poll().put(e);

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerExceptionFactory.java

@@ -42,6 +42,8 @@ public final class SpannerExceptionFactory {
  static final String SESSION_RESOURCE_TYPE = "type.googleapis.com/google.spanner.v1.Session";
  static final String DATABASE_RESOURCE_TYPE =
  "type.googleapis.com/google.spanner.admin.database.v1.Database";
+ static final String INSTANCE_RESOURCE_TYPE =
+ "type.googleapis.com/google.spanner.admin.instance.v1.Instance";
  private static final Metadata.Key<ResourceInfo> KEY_RESOURCE_INFO =
  ProtoUtils.keyForProto(ResourceInfo.getDefaultInstance());
 
@@ -199,6 +201,8 @@ private static SpannerException newSpannerExceptionPreformatted(
  return new SessionNotFoundException(token, message, resourceInfo, cause);
  } else if (resourceInfo.getResourceType().equals(DATABASE_RESOURCE_TYPE)) {
  return new DatabaseNotFoundException(token, message, resourceInfo, cause);
+ } else if (resourceInfo.getResourceType().equals(INSTANCE_RESOURCE_TYPE)) {
+ return new InstanceNotFoundException(token, message, resourceInfo, cause);
  }
  }
  // Fall through to the default.