servlet: add extra 5s to AsyncContext timeout (#12321)

Currently there is a race between 2 tasks scheduled to handle request timeout * servlet AsyncContext timeout * gRPC Context which can cause instability. This change makes the gRPC Context timeout happen first.

Author: panchenko

servlet/src/main/java/io/grpc/servlet/ServletAdapter.java

@@ -22,6 +22,7 @@
 import static java.util.logging.Level.FINE;
 import static java.util.logging.Level.FINEST;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.io.BaseEncoding;
 import io.grpc.Attributes;
 import io.grpc.ExperimentalApi;
@@ -134,10 +135,9 @@ public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOEx
  }
 
  Long timeoutNanos = headers.get(TIMEOUT_KEY);
- if (timeoutNanos == null) {
- timeoutNanos = 0L;
- }
- asyncCtx.setTimeout(TimeUnit.NANOSECONDS.toMillis(timeoutNanos));
+ asyncCtx.setTimeout(timeoutNanos != null
+ ? TimeUnit.NANOSECONDS.toMillis(timeoutNanos) + ASYNC_TIMEOUT_SAFETY_MARGIN
+ : 0);
  StatsTraceContext statsTraceCtx =
  StatsTraceContext.newServerContext(streamTracerFactories, method, headers);
 
@@ -164,6 +164,12 @@ public void doPost(HttpServletRequest req, HttpServletResponse resp) throws IOEx
  asyncCtx.addListener(new GrpcAsyncListener(stream, logId));
  }
 
+ /**
+ * Deadlines are managed via Context, servlet async timeout is not supposed to happen.
+ */
+ @VisibleForTesting
+ static final long ASYNC_TIMEOUT_SAFETY_MARGIN = 5_000;
+
  // This method must use Enumeration and its members, since that is the only way to read headers
  // from the servlet api.
  @SuppressWarnings("JdkObsolete")

servlet/src/test/java/io/grpc/servlet/ServletServerBuilderTest.java

@@ -80,7 +80,7 @@ public void scheduledExecutorService() throws Exception {
  ServletAdapter servletAdapter = serverBuilder.buildServletAdapter();
  servletAdapter.doPost(request, response);
 
- verify(asyncContext).setTimeout(1);
+ verify(asyncContext).setTimeout(1 + ServletAdapter.ASYNC_TIMEOUT_SAFETY_MARGIN);
 
  // The following just verifies that scheduler is populated to the transport.
  // It doesn't matter what tasks (such as handshake timeout and request deadline) are actually