[8.x] Fix internal cluster and single node security tests (#121466)

muted-tests.yml

@@ -395,44 +395,20 @@ tests:
  issue: https://github.com/elastic/elasticsearch/issues/120973
 - class: org.elasticsearch.packaging.test.DockerTests
  issue: https://github.com/elastic/elasticsearch/issues/120978
-- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
- method: testActivateProfileForJWT
- issue: https://github.com/elastic/elasticsearch/issues/120983
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
- method: testProfileIndexAutoCreation
- issue: https://github.com/elastic/elasticsearch/issues/120987
 - class: org.elasticsearch.xpack.security.authc.service.ServiceAccountIT
  method: testAuthenticateShouldNotFallThroughInCaseOfFailure
  issue: https://github.com/elastic/elasticsearch/issues/120902
 - class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT
  method: testReservedStatePersistsOnRestart
  issue: https://github.com/elastic/elasticsearch/issues/120923
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
- method: testSuggestProfilesWithHint
- issue: https://github.com/elastic/elasticsearch/issues/121116
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
  method: test {p0=synonyms/110_synonyms_invalid/Reload index with an invalid synonym rule with lenient set to false}
  issue: https://github.com/elastic/elasticsearch/issues/121117
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
- method: testActivateProfile
- issue: https://github.com/elastic/elasticsearch/issues/121151
 - class: org.elasticsearch.xpack.ml.integration.PyTorchModelIT
  issue: https://github.com/elastic/elasticsearch/issues/121165
-- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
- method: testClientSecretRotation
- issue: https://github.com/elastic/elasticsearch/issues/120985
 - class: org.elasticsearch.xpack.transform.integration.TransformAuditorIT
  method: testAuditorWritesAudits
  issue: https://github.com/elastic/elasticsearch/issues/121241
-- class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
- method: testGrantApiKeyForJWT
- issue: https://github.com/elastic/elasticsearch/issues/121039
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
- method: testSuggestProfileWithData
- issue: https://github.com/elastic/elasticsearch/issues/121258
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
- method: testSuggestProfilesWithName
- issue: https://github.com/elastic/elasticsearch/issues/121022
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
  method: test {p0=nodes.stats/11_indices_metrics/indices mappings exact count test for indices level}
  issue: https://github.com/elastic/elasticsearch/issues/120950
@@ -445,9 +421,6 @@ tests:
 - class: org.elasticsearch.upgrades.VectorSearchIT
  method: testBBQVectorSearch {upgradedNodes=0}
  issue: https://github.com/elastic/elasticsearch/issues/121253
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
- method: testSetEnabled
- issue: https://github.com/elastic/elasticsearch/issues/121183
 - class: org.elasticsearch.upgrades.VectorSearchIT
  method: testBBQVectorSearch {upgradedNodes=1}
  issue: https://github.com/elastic/elasticsearch/issues/121271
@@ -457,20 +430,11 @@ tests:
 - class: org.elasticsearch.xpack.sql.qa.single_node.JdbcDocCsvSpecIT
  method: test {docs.testFilterToday}
  issue: https://github.com/elastic/elasticsearch/issues/121474
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
- method: testGetProfiles
- issue: https://github.com/elastic/elasticsearch/issues/121101
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
- method: testGetUsersWithProfileUidWhenProfileIndexDoesNotExists
- issue: https://github.com/elastic/elasticsearch/issues/121179
 - class: org.elasticsearch.xpack.application.CohereServiceUpgradeIT
  issue: https://github.com/elastic/elasticsearch/issues/121537
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
  method: test {yaml=reference/snapshot-restore/apis/get-snapshot-api/line_488}
  issue: https://github.com/elastic/elasticsearch/issues/121611
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
- method: testUpdateProfileData
- issue: https://github.com/elastic/elasticsearch/issues/121108
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
  method: test {yaml=reference/rest-api/common-options/line_102}
  issue: https://github.com/elastic/elasticsearch/issues/121748
@@ -480,12 +444,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
  method: test {p0=ml/*}
  issue: https://github.com/elastic/elasticsearch/issues/120816
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
- method: testProfileAPIsWhenIndexNotCreated
- issue: https://github.com/elastic/elasticsearch/issues/121096
-- class: org.elasticsearch.xpack.security.authc.service.ServiceAccountSingleNodeTests
- method: testAuthenticateWithServiceFileToken
- issue: https://github.com/elastic/elasticsearch/issues/120988
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
  method: test {yaml=reference/cat/allocation/cat-allocation-example}
  issue: https://github.com/elastic/elasticsearch/issues/121976
@@ -496,12 +454,6 @@ tests:
  issue: https://github.com/elastic/elasticsearch/issues/121291
 - class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
  issue: https://github.com/elastic/elasticsearch/issues/121411
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
- method: testHasPrivileges
- issue: https://github.com/elastic/elasticsearch/issues/121346
-- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
- method: testGetUsersWithProfileUid
- issue: https://github.com/elastic/elasticsearch/issues/121483
 - class: org.elasticsearch.xpack.application.FullClusterRestartIT
  issue: https://github.com/elastic/elasticsearch/issues/121935
 - class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
@@ -518,4 +470,4 @@ tests:
  issue: https://github.com/elastic/elasticsearch/issues/122683
 - class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
  method: testStopQueryLocal
- issue: https://github.com/elastic/elasticsearch/issues/121672
+ issue: https://github.com/elastic/elasticsearch/issues/121672

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java

@@ -7,9 +7,16 @@
 package org.elasticsearch.test;
 
 import org.apache.http.HttpHost;
+import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.action.admin.cluster.node.info.NodeInfo;
 import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
+import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
+import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
+import org.elasticsearch.action.support.ActiveShardCount;
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.client.RestClientBuilder;
@@ -27,6 +34,7 @@
 import org.elasticsearch.license.LicenseSettings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
+import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
 import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices;
 import org.elasticsearch.xpack.security.LocalStateSecurity;
 import org.elasticsearch.xpack.security.support.SecurityMigrations;
@@ -45,9 +53,12 @@
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.test.SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING;
+import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue;
 import static org.elasticsearch.xpack.security.support.SecurityIndexManager.getMigrationVersionFromIndexMetadata;
+import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS;
 import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.is;
 
 /**
  * A test that starts a single node with security enabled. This test case allows for customization
@@ -82,6 +93,13 @@ public static void destroyDefaultSettings() {
  tearDownRestClient();
  }
 
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ deleteSecurityIndexIfExists();
+ createSecurityIndexWithWaitForActiveShards();
+ }
+
  @Override
  public void tearDown() throws Exception {
  awaitSecurityMigration();
@@ -100,7 +118,7 @@ private boolean isMigrationComplete(ClusterState state) {
  return getMigrationVersionFromIndexMetadata(indexMetadata) == SecurityMigrations.MIGRATIONS_BY_VERSION.lastKey();
  }
 
- private void awaitSecurityMigration() {
+ protected void awaitSecurityMigration() {
  final var latch = new CountDownLatch(1);
  ClusterService clusterService = getInstanceFromNode(ClusterService.class);
  clusterService.addListener((event) -> {
@@ -362,4 +380,40 @@ private static RestClient createRestClient(
  }
  return builder.build();
  }
+
+ protected void deleteSecurityIndexIfExists() {
+ // delete the security index, if it exist
+ GetIndexRequest getIndexRequest = new GetIndexRequest();
+ getIndexRequest.indices(SECURITY_MAIN_ALIAS);
+ getIndexRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
+ GetIndexResponse getIndexResponse = client().admin().indices().getIndex(getIndexRequest).actionGet();
+ if (getIndexResponse.getIndices().length > 0) {
+ assertThat(getIndexResponse.getIndices().length, is(1));
+ assertThat(getIndexResponse.getIndices()[0], is(TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7));
+
+ // Security migration needs to finish before deleting the index
+ awaitSecurityMigration();
+ DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(getIndexResponse.getIndices());
+ assertAcked(client().admin().indices().delete(deleteIndexRequest).actionGet());
+ }
+ }
+
+ protected void createSecurityIndexWithWaitForActiveShards() {
+ final Client client = client().filterWithHeader(
+ Collections.singletonMap(
+ "Authorization",
+ UsernamePasswordToken.basicAuthHeaderValue(
+ SecuritySettingsSource.ES_TEST_ROOT_USER,
+ SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING
+ )
+ )
+ );
+ CreateIndexRequest createIndexRequest = new CreateIndexRequest(SECURITY_MAIN_ALIAS).waitForActiveShards(ActiveShardCount.ALL)
+ .masterNodeTimeout(TEST_REQUEST_TIMEOUT);
+ try {
+ client.admin().indices().create(createIndexRequest).actionGet();
+ } catch (ResourceAlreadyExistsException e) {
+ logger.info("Security index already exists, ignoring.", e);
+ }
+ }
 }

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/esnative/ReservedRealmElasticAutoconfigIntegTests.java

@@ -8,17 +8,10 @@
 package org.elasticsearch.xpack.security.authc.esnative;
 
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
-import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
-import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.ResponseException;
-import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
-import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.SecureString;
@@ -29,14 +22,9 @@
 import org.elasticsearch.xpack.core.security.action.user.PutUserRequest;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken;
-import org.elasticsearch.xpack.core.security.test.TestRestrictedIndices;
 import org.junit.BeforeClass;
 
-import java.util.concurrent.CountDownLatch;
-
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
-import static org.elasticsearch.xpack.core.security.action.UpdateIndexMigrationVersionAction.MIGRATION_VERSION_CUSTOM_KEY;
-import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS;
 import static org.hamcrest.Matchers.is;
 
 public class ReservedRealmElasticAutoconfigIntegTests extends SecuritySingleNodeTestCase {
@@ -70,46 +58,10 @@ protected SecureString getBootstrapPassword() {
  return null; // no bootstrap password for this test
  }
 
- private boolean isMigrationComplete(ClusterState state) {
- IndexMetadata indexMetadata = state.metadata().getIndices().get(TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7);
- return indexMetadata != null && indexMetadata.getCustomData(MIGRATION_VERSION_CUSTOM_KEY) != null;
- }
-
- private void awaitSecurityMigrationRanOnce() {
- final var latch = new CountDownLatch(1);
- ClusterService clusterService = getInstanceFromNode(ClusterService.class);
- clusterService.addListener((event) -> {
- if (isMigrationComplete(event.state())) {
- latch.countDown();
- }
- });
- if (isMigrationComplete(clusterService.state())) {
- latch.countDown();
- }
- safeAwait(latch);
- }
-
- private void deleteSecurityIndex() {
- // delete the security index, if it exist
- GetIndexRequest getIndexRequest = new GetIndexRequest();
- getIndexRequest.indices(SECURITY_MAIN_ALIAS);
- getIndexRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
- GetIndexResponse getIndexResponse = client().admin().indices().getIndex(getIndexRequest).actionGet();
- if (getIndexResponse.getIndices().length > 0) {
- assertThat(getIndexResponse.getIndices().length, is(1));
- assertThat(getIndexResponse.getIndices()[0], is(TestRestrictedIndices.INTERNAL_SECURITY_MAIN_INDEX_7));
-
- // Security migration needs to finish before deleting the index
- awaitSecurityMigrationRanOnce();
- DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(getIndexResponse.getIndices());
- assertAcked(client().admin().indices().delete(deleteIndexRequest).actionGet());
- }
- }
-
  public void testAutoconfigFailedPasswordPromotion() throws Exception {
  try {
  // .security index is created automatically on node startup so delete the security index first
- deleteSecurityIndex();
+ deleteSecurityIndexIfExists();
  // prevents the .security index from being created automatically (after elastic user authentication)
  ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(
  TEST_REQUEST_TIMEOUT,
@@ -176,7 +128,7 @@ public void testAutoconfigSucceedsAfterPromotionFailure() throws Exception {
  putUserRequest.roles(Strings.EMPTY_ARRAY);
  client().execute(PutUserAction.INSTANCE, putUserRequest).get();
  // Security migration needs to finish before making the cluster read only
- awaitSecurityMigrationRanOnce();
+ awaitSecurityMigration();
 
  // but then make the cluster read-only
  ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileIntegTests.java

@@ -99,13 +99,28 @@
 
 public class ProfileIntegTests extends AbstractProfileIntegTestCase {
 
+ protected static final String ANONYMOUS_ROLE = "anonymous_role";
+
+ @Override
+ protected String configRoles() {
+ return super.configRoles()
+ + "\n"
+ + ANONYMOUS_ROLE
+ + ":\n"
+ + " cluster:\n"
+ + " - 'manage_own_api_key'\n"
+ + " - 'manage_token'\n"
+ + " - 'manage_service_account'\n"
+ + " - 'monitor'\n";
+ }
+
  @Override
  protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
  final Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings));
  // This setting tests that the setting is registered
  builder.put("xpack.security.authc.domains.my_domain.realms", "file");
  // enable anonymous
- builder.putList(AnonymousUser.ROLES_SETTING.getKey(), RAC_ROLE);
+ builder.putList(AnonymousUser.ROLES_SETTING.getKey(), ANONYMOUS_ROLE);
  return builder.build();
  }