Disable queryable built-in roles feature for core and datastream YAML tests (#121541)

This PR disables the "queryable built-in roles" feature for the `CoreWithSecurityClientYamlTestSuiteIT` and `DataStreamsClientYamlTestSuiteIT` YAML test suites. The feature was enabled by default in the #120323 PR, which asynchronously creates the `.security` index after cluster formation and indexes all built-in roles. The asynchronous creation of the `.security` index introduces non-deterministic behavior in our YAML tests. Since these test suites are not intended to verify the queryable built-in roles functionality, having the feature enabled introduced flakiness and unnecessary complexity to handle `.security` in existing tests. These tests would have to exclude the `.security` index in some way (by adjusting permissions or API calls), and in the end cleanup (delete) the `.security` index. This simply adds overhead without much gain. The feature is already test covered by `XPackRestIT` and other integration/REST tests, disabling it here would not compromise test coverage. Instead, it ensures these suites remain deterministic and focused on the behaviors they were designed to verify. Resolves #121536 Resolves #121513 Resolves #121484 Resolves #121478 Resolves #121290 Resolves #121246 Resolves #121242 Resolves #121238 Resolves #121186 Resolves #121131 Resolves #121130 Resolves #121128 Resolves #121014 Resolves #120965 Resolves #120920 Resolves #120890

Author: slobodanadamovic

modules/data-streams/src/yamlRestTest/java/org/elasticsearch/datastreams/DataStreamsClientYamlTestSuiteIT.java

@@ -49,7 +49,8 @@ private static ElasticsearchCluster createCluster() {
  .feature(FAILURE_STORE_ENABLED)
  .setting("xpack.security.enabled", "true")
  .keystore("bootstrap.password", "x-pack-test-password")
- .user("x_pack_rest_user", "x-pack-test-password");
+ .user("x_pack_rest_user", "x-pack-test-password")
+ .systemProperty("es.queryable_built_in_roles_enabled", "false");
  if (initTestSeed().nextBoolean()) {
  clusterBuilder.setting("xpack.license.self_generated.type", "trial");
  }

muted-tests.yml

@@ -214,9 +214,6 @@ tests:
  issue: https://github.com/elastic/elasticsearch/issues/120810
 - class: org.elasticsearch.indices.mapping.UpdateMappingIntegrationIT
  issue: https://github.com/elastic/elasticsearch/issues/116126
-- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
- method: test {p0=data_stream/140_data_stream_aliases/Create data stream aliases using wildcard expression}
- issue: https://github.com/elastic/elasticsearch/issues/120890
 - class: org.elasticsearch.xpack.security.authc.service.ServiceAccountIT
  method: testAuthenticateShouldNotFallThroughInCaseOfFailure
  issue: https://github.com/elastic/elasticsearch/issues/120902
@@ -226,9 +223,6 @@ tests:
 - class: org.elasticsearch.packaging.test.DockerTests
  method: test140CgroupOsStatsAreAvailable
  issue: https://github.com/elastic/elasticsearch/issues/120914
-- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
- method: test {p0=data_stream/140_data_stream_aliases/Create data stream alias}
- issue: https://github.com/elastic/elasticsearch/issues/120920
 - class: org.elasticsearch.xpack.security.FileSettingsRoleMappingsRestartIT
  method: testReservedStatePersistsOnRestart
  issue: https://github.com/elastic/elasticsearch/issues/120923
@@ -244,12 +238,6 @@ tests:
 - class: org.elasticsearch.action.search.SearchProgressActionListenerIT
  method: testSearchProgressWithQuery
  issue: https://github.com/elastic/elasticsearch/issues/120994
-- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
- method: test {p0=data_stream/80_resolve_index_data_streams/Resolve index with hidden and closed indices}
- issue: https://github.com/elastic/elasticsearch/issues/120965
-- class: org.elasticsearch.datastreams.DataStreamsClientYamlTestSuiteIT
- method: test {p0=data_stream/140_data_stream_aliases/Create data stream alias with filter}
- issue: https://github.com/elastic/elasticsearch/issues/121014
 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
  method: testSuggestProfilesWithName
  issue: https://github.com/elastic/elasticsearch/issues/121022
@@ -271,9 +259,6 @@ tests:
 - class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
  method: testActivateProfileForJWT
  issue: https://github.com/elastic/elasticsearch/issues/120983
-- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
- method: test {yaml=cluster.health/20_request_timeout/cluster health request timeout waiting for active shards}
- issue: https://github.com/elastic/elasticsearch/issues/121130
 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
  method: testProfileIndexAutoCreation
  issue: https://github.com/elastic/elasticsearch/issues/120987
@@ -291,21 +276,9 @@ tests:
 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
  method: testSetEnabled
  issue: https://github.com/elastic/elasticsearch/issues/121183
-- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
- method: test {yaml=cat.aliases/10_basic/Simple alias}
- issue: https://github.com/elastic/elasticsearch/issues/121186
 - class: org.elasticsearch.xpack.ml.integration.ClassificationIT
  method: testWithDatastreams
  issue: https://github.com/elastic/elasticsearch/issues/121236
-- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
- method: test {yaml=nodes.stats/11_indices_metrics/Metric - blank for indices mappings}
- issue: https://github.com/elastic/elasticsearch/issues/121238
-- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
- method: test {yaml=indices.get_alias/10_basic/Get aliases via /_alias/_all}
- issue: https://github.com/elastic/elasticsearch/issues/121242
-- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
- method: test {yaml=cluster.stats/10_basic/Sparse vector stats}
- issue: https://github.com/elastic/elasticsearch/issues/121246
 - class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityEsqlIT
  method: testCrossClusterAsyncQueryStop
  issue: https://github.com/elastic/elasticsearch/issues/121249
@@ -338,9 +311,6 @@ tests:
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
  method: test {yaml=reference/index-modules/slowlog/line_102}
  issue: https://github.com/elastic/elasticsearch/issues/121288
-- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
- method: test {yaml=indices.get_alias/10_basic/Get aliases via /*/_alias/}
- issue: https://github.com/elastic/elasticsearch/issues/121290
 - class: org.elasticsearch.env.NodeEnvironmentTests
  method: testGetBestDowngradeVersion
  issue: https://github.com/elastic/elasticsearch/issues/121316
@@ -379,15 +349,9 @@ tests:
 - class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
  method: testGrantApiKeyForJWT
  issue: https://github.com/elastic/elasticsearch/issues/121039
-- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
- method: test {yaml=cluster.health/10_basic/cluster health basic test}
- issue: https://github.com/elastic/elasticsearch/issues/121478
 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
  method: testGetUsersWithProfileUid
  issue: https://github.com/elastic/elasticsearch/issues/121483
-- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
- method: test {yaml=cat.aliases/10_basic/Empty cluster}
- issue: https://github.com/elastic/elasticsearch/issues/121484
 - class: org.elasticsearch.xpack.transform.checkpoint.TransformCCSCanMatchIT
  method: testTransformLifecycle_RangeQueryThatMatchesNoShards
  issue: https://github.com/elastic/elasticsearch/issues/121480
@@ -403,9 +367,6 @@ tests:
 - class: org.elasticsearch.xpack.core.ilm.SetSingleNodeAllocateStepTests
  method: testPerformActionSomeShardsOnlyOnNewNodesButNewNodesInvalidAttrs
  issue: https://github.com/elastic/elasticsearch/issues/121495
-- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
- method: test {yaml=cat.aliases/40_hidden/Test cat aliases output with a visible index with a hidden alias}
- issue: https://github.com/elastic/elasticsearch/issues/121128
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
  method: test {p0=search.vectors/42_knn_search_int4_flat/Vector similarity with filter only}
  issue: https://github.com/elastic/elasticsearch/issues/121412
@@ -414,12 +375,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.ClassificationIT
  method: testDependentVariableIsAliasToKeyword
  issue: https://github.com/elastic/elasticsearch/issues/121492
-- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
- method: test {yaml=cat.aliases/10_basic/Complex alias}
- issue: https://github.com/elastic/elasticsearch/issues/121513
-- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
- method: test {yaml=snapshot.create/10_basic/Create a snapshot for missing index}
- issue: https://github.com/elastic/elasticsearch/issues/121536
 - class: org.elasticsearch.xpack.esql.action.CrossClusterQueryUnavailableRemotesIT
  method: testRemoteOnlyCCSAgainstDisconnectedRemoteWithSkipUnavailableTrue
  issue: https://github.com/elastic/elasticsearch/issues/121578

x-pack/plugin/ml/qa/ml-with-security/build.gradle

@@ -258,4 +258,5 @@ testClusters.configureEach {
  user username: "no_ml", password: "x-pack-test-password", role: "minimal"
  setting 'xpack.license.self_generated.type', 'trial'
  setting 'xpack.security.enabled', 'true'
+ systemProperty 'es.queryable_built_in_roles_enabled', 'false'
 }

x-pack/qa/core-rest-tests-with-security/src/yamlRestTest/java/org/elasticsearch/xpack/security/CoreWithSecurityClientYamlTestSuiteIT.java

@@ -46,6 +46,7 @@ public class CoreWithSecurityClientYamlTestSuiteIT extends ESClientYamlSuiteTest
  .setting("xpack.ml.enabled", "false")
  .setting("xpack.license.self_generated.type", "trial")
  .setting("xpack.security.autoconfiguration.enabled", "false")
+ .systemProperty("es.queryable_built_in_roles_enabled", "false")
  .user(USER, PASS)
  .feature(FeatureFlag.TIME_SERIES_MODE)
  .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)

x-pack/qa/rolling-upgrade/build.gradle

@@ -47,7 +47,6 @@ buildParams.bwcVersions.withWireCompatible { bwcVersion, baseName ->
  testDistribution = "DEFAULT"
  versions = [oldVersion, project.version]
  numberOfNodes = 3
- systemProperty 'es.queryable_built_in_roles_enabled', 'true'
  systemProperty 'ingest.geoip.downloader.enabled.default', 'true'
  //we don't want to hit real service from each test
  systemProperty 'ingest.geoip.downloader.endpoint.default', 'http://invalid.endpoint'