improve reliability of SslStream tests with failing certificate validation (#43570)

* improve reliability of SslStream tests with failing certificate validation * Update src/libraries/System.Net.Security/src/System/Net/Security/SslStream.Implementation.cs Co-authored-by: Stephen Toub <stoub@microsoft.com> * fix failing ALPN test on old OpenSSL Co-authored-by: Stephen Toub <stoub@microsoft.com>

Author: wfurt

src/libraries/System.Net.Security/src/System/Net/Security/SslStream.Implementation.cs

@@ -362,7 +362,7 @@ private async Task ForceAuthenticationAsync<TIOAdapter>(TIOAdapter adapter, bool
  payload = message.Payload;
  size = message.Size;
  }
- else if (message.Failed && _lastFrame.Header.Type == TlsContentType.Handshake)
+ else if (message.Failed && (_lastFrame.Header.Type == TlsContentType.Handshake || _lastFrame.Header.Type == TlsContentType.ChangeCipherSpec))
  {
  // If we failed without OS sending out alert, inject one here to be consistent across platforms.
  payload = TlsFrameHelper.CreateAlertFrame(_lastFrame.Header.Version, TlsAlertDescription.ProtocolVersion);
@@ -562,7 +562,8 @@ private ProtocolToken ProcessBlob(int frameSize)
  }
 
  frameSize = nextHeader.Length + TlsFrameHelper.HeaderSize;
- if (nextHeader.Type == TlsContentType.AppData || frameSize > _handshakeBuffer.ActiveLength)
+ // Can process more handshake frames in single step, but we should avoid processing too much so as to preserve API boundary between handshake and I/O.
+ if ((nextHeader.Type != TlsContentType.Handshake && nextHeader.Type != TlsContentType.ChangeCipherSpec) || frameSize > _handshakeBuffer.ActiveLength)
  {
  // We don't have full frame left or we already have app data which needs to be processed by decrypt.
  break;

src/libraries/System.Net.Security/tests/FunctionalTests/SslStreamAlertsTest.cs

@@ -29,14 +29,16 @@ public async Task SslStream_StreamToStream_HandshakeAlert_Ok()
  using (var server = new SslStream(stream2, true, FailClientCertificate))
  using (X509Certificate2 certificate = Configuration.Certificates.GetServerCertificate())
  {
+ int timeout = TestConfiguration.PassingTestTimeoutMilliseconds;
+
  Task serverAuth = server.AuthenticateAsServerAsync(certificate);
- await client.AuthenticateAsClientAsync(certificate.GetNameInfo(X509NameType.SimpleName, false));
+ await client.AuthenticateAsClientAsync(certificate.GetNameInfo(X509NameType.SimpleName, false)).TimeoutAfter(timeout);
 
  byte[] buffer = new byte[1024];
 
  // Schannel semantics require that Decrypt is called to receive an alert.
  await client.WriteAsync(buffer, 0, buffer.Length);
- var exception = await Assert.ThrowsAsync<IOException>(() => client.ReadAsync(buffer, 0, buffer.Length));
+ var exception = await Assert.ThrowsAsync<IOException>(() => client.ReadAsync(buffer, 0, buffer.Length)).TimeoutAfter(timeout);
 
  Assert.IsType<Win32Exception>(exception.InnerException);
  var win32ex = (Win32Exception)exception.InnerException;
@@ -45,7 +47,7 @@ public async Task SslStream_StreamToStream_HandshakeAlert_Ok()
  // https://msdn.microsoft.com/en-us/library/windows/desktop/dd721886(v=vs.85).aspx
  Assert.Equal(SEC_E_CERT_UNKNOWN, unchecked((uint)win32ex.NativeErrorCode));
 
- await Assert.ThrowsAsync<AuthenticationException>(() => serverAuth);
+ await Assert.ThrowsAsync<AuthenticationException>(() => serverAuth).TimeoutAfter(timeout);
 
  await Assert.ThrowsAsync<AuthenticationException>(() => server.WriteAsync(buffer, 0, buffer.Length));
  await Assert.ThrowsAsync<AuthenticationException>(() => server.ReadAsync(buffer, 0, buffer.Length));