Validate certificate identity from cross cluster creds

Author: jfreden

x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityCrossClusterApiKeySigningIT.java

@@ -8,7 +8,6 @@
 package org.elasticsearch.xpack.remotecluster;
 
 import io.netty.handler.codec.http.HttpMethod;
-
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
@@ -25,6 +24,7 @@
 import org.junit.rules.TestRule;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Locale;
@@ -38,7 +38,7 @@
 
 public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRemoteClusterSecurityTestCase {
 
- private static final AtomicReference<Map<String, Object>> API_KEY_MAP_REF = new AtomicReference<>();
+ private static final AtomicReference<Map<String, Object>> MY_REMOTE_API_KEY_MAP_REF = new AtomicReference<>();
 
  static {
  fulfillingCluster = ElasticsearchCluster.local()
@@ -49,9 +49,14 @@ public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRe
  .setting("xpack.security.remote_cluster_server.ssl.enabled", "true")
  .setting("xpack.security.remote_cluster_server.ssl.key", "remote-cluster.key")
  .setting("xpack.security.remote_cluster_server.ssl.certificate", "remote-cluster.crt")
+ .setting("xpack.security.audit.enabled", "true")
+ .setting(
+ "xpack.security.audit.logfile.events.include",
+ "[authentication_success, authentication_failed, access_denied, access_granted]"
+ )
  .configFile("signing_ca.crt", Resource.fromClasspath("signing/root.crt"))
- .setting("cluster.remote.signing.certificate_authorities", "signing_ca.crt")
  .keystore("xpack.security.remote_cluster_server.ssl.secure_key_passphrase", "remote-cluster-password")
+ .setting("logger.org.elasticsearch.xpack.security", "trace") // useful for human debugging
  .build();
 
  queryCluster = ElasticsearchCluster.local()
@@ -60,22 +65,25 @@ public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRe
  .setting("xpack.security.remote_cluster_client.ssl.enabled", "true")
  .setting("xpack.security.remote_cluster_client.ssl.certificate_authorities", "remote-cluster-ca.crt")
  .configFile("signing.crt", Resource.fromClasspath("signing/signing.crt"))
- .setting("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt")
  .configFile("signing.key", Resource.fromClasspath("signing/signing.key"))
- .setting("cluster.remote.my_remote_cluster.signing.key", "signing.key")
  .keystore("cluster.remote.my_remote_cluster.credentials", () -> {
- if (API_KEY_MAP_REF.get() == null) {
- final Map<String, Object> apiKeyMap = createCrossClusterAccessApiKey("""
+ if (MY_REMOTE_API_KEY_MAP_REF.get() == null) {
+ final var accessJson = """
  {
  "search": [
  {
  "names": ["index*", "not_found_index"]
  }
  ]
- }""");
- API_KEY_MAP_REF.set(apiKeyMap);
+ }""";
+ MY_REMOTE_API_KEY_MAP_REF.set(
+ createCrossClusterAccessApiKey(
+ accessJson,
+ randomFrom("CN=instance", "^CN=instance$", "(?i)^CN=instance$", "^CN=[A-Za-z0-9_]+$")
+ )
+ );
  }
- return (String) API_KEY_MAP_REF.get().get("encoded");
+ return (String) MY_REMOTE_API_KEY_MAP_REF.get().get("encoded");
  })
  .keystore("cluster.remote.invalid_remote.credentials", randomEncodedApiKey())
  .build();
@@ -86,33 +94,102 @@ public class RemoteClusterSecurityCrossClusterApiKeySigningIT extends AbstractRe
  public static TestRule clusterRule = RuleChain.outerRule(fulfillingCluster).around(queryCluster);
 
  public void testCrossClusterSearchWithCrossClusterApiKeySigning() throws Exception {
- indexTestData();
- assertCrossClusterSearchSuccessfulWithResult();
+ updateClusterSettings(
+ Settings.builder()
+ .put("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt")
+ .put("cluster.remote.my_remote_cluster.signing.key", "signing.key")
+ .build()
+ );
 
- // Change the CA to something that doesn't trust the signing cert
  updateClusterSettingsFulfillingCluster(
- Settings.builder().put("cluster.remote.signing.certificate_authorities", "transport-ca.crt").build()
+ Settings.builder().put("cluster.remote.signing.certificate_authorities", "signing_ca.crt").build()
  );
- assertCrossClusterAuthFail();
 
- // Update settings on query cluster to ignore unavailable remotes
- updateClusterSettings(Settings.builder().put("cluster.remote.my_remote_cluster.skip_unavailable", Boolean.toString(true)).build());
+ indexTestData();
 
- assertCrossClusterSearchSuccessfulWithoutResult();
+ // Make sure we can search if cert trusted
+ {
+ assertCrossClusterSearchSuccessfulWithResult();
+ }
 
- // TODO add test for certificate identity configured for API key but no signature provided (should 401)
+ // Test CA that does not trust cert
+ {
+ // Change the CA to something that doesn't trust the signing cert
+ updateClusterSettingsFulfillingCluster(
+ Settings.builder().put("cluster.remote.signing.certificate_authorities", "transport-ca.crt").build()
+ );
+ assertCrossClusterAuthFail("Failed to verify cross cluster api key signature certificate from [(");
+
+ // Change the CA to the default trust store
+ updateClusterSettingsFulfillingCluster(Settings.builder().putNull("cluster.remote.signing.certificate_authorities").build());
+ assertCrossClusterAuthFail("Failed to verify cross cluster api key signature certificate from [(");
+
+ // Update settings on query cluster to ignore unavailable remotes
+ updateClusterSettings(
+ Settings.builder().put("cluster.remote.my_remote_cluster.skip_unavailable", Boolean.toString(true)).build()
+ );
+ assertCrossClusterSearchSuccessfulWithoutResult();
 
- // TODO add test for certificate identity not configured for API key but signature provided (should 200)
+ // Reset skip_unavailable
+ updateClusterSettings(
+ Settings.builder().put("cluster.remote.my_remote_cluster.skip_unavailable", Boolean.toString(false)).build()
+ );
 
- // TODO add test for certificate identity not configured for API key but wrong signature provided (should 401)
+ // Reset ca cert
+ updateClusterSettingsFulfillingCluster(
+ Settings.builder().put("cluster.remote.signing.certificate_authorities", "signing_ca.crt").build()
+ );
+ // Confirm reset was successful
+ assertCrossClusterSearchSuccessfulWithResult();
+ }
+
+ // Test no signature provided
+ {
+ updateClusterSettings(
+ Settings.builder()
+ .putNull("cluster.remote.my_remote_cluster.signing.certificate")
+ .putNull("cluster.remote.my_remote_cluster.signing.key")
+ .build()
+ );
+ assertCrossClusterAuthFail("Expected signature for cross cluster api key but no signature provided");
 
- // TODO add test for certificate identity regex matching (should 200)
+ // Reset
+ updateClusterSettings(
+ Settings.builder()
+ .put("cluster.remote.my_remote_cluster.signing.certificate", "signing.crt")
+ .put("cluster.remote.my_remote_cluster.signing.key", "signing.key")
+ .build()
+ );
+ }
+
+ // Test API key without certificate identity and send signature anyway
+ {
+ final var accessJson = """
+ {
+ "search": [
+ {
+ "names": ["index*", "not_found_index"]
+ }
+ ]
+ }""";
+ MY_REMOTE_API_KEY_MAP_REF.set(createCrossClusterAccessApiKey(accessJson));
+ assertCrossClusterSearchSuccessfulWithResult();
+
+ // Change the CA to the default trust store to make sure untrusted signature fails auth even if it's not required
+ updateClusterSettingsFulfillingCluster(Settings.builder().putNull("cluster.remote.signing.certificate_authorities").build());
+ assertCrossClusterAuthFail("Failed to verify cross cluster api key signature certificate from [(");
+
+ // Reset
+ updateClusterSettingsFulfillingCluster(
+ Settings.builder().put("cluster.remote.signing.certificate_authorities", "signing_ca.crt").build()
+ );
+ }
  }
 
- private void assertCrossClusterAuthFail() {
+ private void assertCrossClusterAuthFail(String expectedMessage) {
  var responseException = assertThrows(ResponseException.class, () -> simpleCrossClusterSearch(randomBoolean()));
  assertThat(responseException.getResponse().getStatusLine().getStatusCode(), equalTo(401));
- assertThat(responseException.getMessage(), containsString("Failed to verify cross cluster api key signature certificate from [("));
+ assertThat(responseException.getMessage(), containsString(expectedMessage));
  }
 
  private void assertCrossClusterSearchSuccessfulWithoutResult() throws IOException {
@@ -227,4 +304,25 @@ private Response performRequestWithRemoteAccessUser(final Request request) throw
  request.setOptions(RequestOptions.DEFAULT.toBuilder().addHeader("Authorization", basicAuthHeaderValue(REMOTE_SEARCH_USER, PASS)));
  return client().performRequest(request);
  }
+
+ protected static Map<String, Object> createCrossClusterAccessApiKey(String accessJson, String certificateIdentity) {
+ initFulfillingClusterClient();
+ final var createCrossClusterApiKeyRequest = new Request("POST", "/_security/cross_cluster/api_key");
+ createCrossClusterApiKeyRequest.setJsonEntity(Strings.format("""
+ {
+ "name": "cross_cluster_access_key",
+ "certificate_identity": "%s",
+ "access": %s
+ }""", certificateIdentity, accessJson));
+ try {
+ final Response createCrossClusterApiKeyResponse = performRequestWithAdminUser(
+ fulfillingClusterClient,
+ createCrossClusterApiKeyRequest
+ );
+ assertOK(createCrossClusterApiKeyResponse);
+ return responseAsMap(createCrossClusterApiKeyResponse);
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
 }

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -141,6 +141,7 @@
 import java.util.function.Consumer;
 import java.util.function.Function;
 import java.util.function.Supplier;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.SecureRandomUtils.getBase64SecureRandomString;
@@ -1396,7 +1397,7 @@ void validateApiKeyCredentials(
  if (result.success) {
  if (result.verify(credentials.getKey())) {
  // move on
- validateApiKeyTypeAndExpiration(apiKeyDoc, credentials, clock, listener);
+ completeApiKeyAuthentication(apiKeyDoc, credentials, clock, listener);
  } else {
  listener.onResponse(
  AuthenticationResult.unsuccessful("invalid credentials for API key [" + credentials.getId() + "]", null)
@@ -1416,7 +1417,7 @@ void validateApiKeyCredentials(
  listenableCacheEntry.onResponse(new CachedApiKeyHashResult(verified, credentials.getKey()));
  if (verified) {
  // move on
- validateApiKeyTypeAndExpiration(apiKeyDoc, credentials, clock, listener);
+ completeApiKeyAuthentication(apiKeyDoc, credentials, clock, listener);
  } else {
  listener.onResponse(
  AuthenticationResult.unsuccessful("invalid credentials for API key [" + credentials.getId() + "]", null)
@@ -1439,7 +1440,7 @@ void validateApiKeyCredentials(
  verifyKeyAgainstHash(apiKeyDoc.hash, credentials, ActionListener.wrap(verified -> {
  if (verified) {
  // move on
- validateApiKeyTypeAndExpiration(apiKeyDoc, credentials, clock, listener);
+ completeApiKeyAuthentication(apiKeyDoc, credentials, clock, listener);
  } else {
  listener.onResponse(
  AuthenticationResult.unsuccessful("invalid credentials for API key [" + credentials.getId() + "]", null)
@@ -1471,7 +1472,7 @@ Cache<String, BytesReference> getRoleDescriptorsBytesCache() {
  }
 
  // package-private for testing
- static void validateApiKeyTypeAndExpiration(
+ static void completeApiKeyAuthentication(
  ApiKeyDoc apiKeyDoc,
  ApiKeyCredentials credentials,
  Clock clock,
@@ -1491,6 +1492,27 @@ static void validateApiKeyTypeAndExpiration(
  return;
  }
 
+ if (apiKeyDoc.certificateIdentity != null) {
+ if (credentials.getCertificateIdentity() == null) {
+ listener.onResponse(
+ AuthenticationResult.terminate("Expected signature for cross cluster api key but no signature provided")
+ );
+ return;
+ }
+ if (validateCertificateIdentity(credentials.getCertificateIdentity(), apiKeyDoc.certificateIdentity) == false) {
+ listener.onResponse(
+ AuthenticationResult.terminate(
+ Strings.format(
+ "DN from provided certificate [%s] does no match API Key certificate identity pattern [%s]",
+ credentials.getCertificateIdentity(),
+ apiKeyDoc.certificateIdentity
+ )
+ )
+ );
+ return;
+ }
+ }
+
  if (apiKeyDoc.expirationTime == -1 || Instant.ofEpochMilli(apiKeyDoc.expirationTime).isAfter(clock.instant())) {
  final String principal = Objects.requireNonNull((String) apiKeyDoc.creator.get("principal"));
  final String fullName = (String) apiKeyDoc.creator.get("full_name");
@@ -1515,22 +1537,31 @@ static void validateApiKeyTypeAndExpiration(
  }
  }
 
+ private static boolean validateCertificateIdentity(String certificateIdentity, String certificateIdentityPattern) {
+ logger.trace("Validating certificate identity [{}] against [{}]", certificateIdentity, certificateIdentityPattern);
+ return Pattern.compile(certificateIdentityPattern).matcher(certificateIdentity).matches();
+ }
+
  ApiKeyCredentials parseCredentialsFromApiKeyString(SecureString apiKeyString) {
  if (false == isEnabled()) {
  return null;
  }
- return parseApiKey(apiKeyString, ApiKey.Type.REST);
+ return parseApiKey(apiKeyString, null, ApiKey.Type.REST);
  }
 
- static ApiKeyCredentials getCredentialsFromHeader(final String header, ApiKey.Type expectedType) {
- return parseApiKey(Authenticator.extractCredentialFromHeaderValue(header, "ApiKey"), expectedType);
+ static ApiKeyCredentials getCredentialsFromHeader(final String header, @Nullable String certificateIdentity, ApiKey.Type expectedType) {
+ return parseApiKey(Authenticator.extractCredentialFromHeaderValue(header, "ApiKey"), certificateIdentity, expectedType);
  }
 
  public static String withApiKeyPrefix(final String encodedApiKey) {
  return "ApiKey " + encodedApiKey;
  }
 
- private static ApiKeyCredentials parseApiKey(SecureString apiKeyString, ApiKey.Type expectedType) {
+ private static ApiKeyCredentials parseApiKey(
+ SecureString apiKeyString,
+ @Nullable String certificateIdentity,
+ ApiKey.Type expectedType
+ ) {
  if (apiKeyString != null) {
  final byte[] decodedApiKeyCredBytes = Base64.getDecoder().decode(CharArrays.toUtf8Bytes(apiKeyString.getChars()));
  char[] apiKeyCredChars = null;
@@ -1554,7 +1585,8 @@ private static ApiKeyCredentials parseApiKey(SecureString apiKeyString, ApiKey.T
  return new ApiKeyCredentials(
  new String(Arrays.copyOfRange(apiKeyCredChars, 0, colonIndex)),
  new SecureString(Arrays.copyOfRange(apiKeyCredChars, secretStartPos, apiKeyCredChars.length)),
- expectedType
+ expectedType,
+ certificateIdentity
  );
  } finally {
  if (apiKeyCredChars != null) {
@@ -1671,11 +1703,17 @@ public static final class ApiKeyCredentials implements AuthenticationToken, Clos
  private final String id;
  private final SecureString key;
  private final ApiKey.Type expectedType;
+ private final String certificateIdentity;
 
  public ApiKeyCredentials(String id, SecureString key, ApiKey.Type expectedType) {
+ this(id, key, expectedType, null);
+ }
+
+ public ApiKeyCredentials(String id, SecureString key, ApiKey.Type expectedType, @Nullable String certificateIdentity) {
  this.id = id;
  this.key = key;
  this.expectedType = expectedType;
+ this.certificateIdentity = certificateIdentity;
  }
 
  String getId() {
@@ -1709,6 +1747,10 @@ public void clearCredentials() {
  public ApiKey.Type getExpectedType() {
  return expectedType;
  }
+
+ public String getCertificateIdentity() {
+ return certificateIdentity;
+ }
  }
 
  private static class ApiKeyLoggingDeprecationHandler implements DeprecationHandler {