Include search idle info to shard stats

docs/changelog/95740.yaml

@@ -0,0 +1,6 @@
+pr: 95740
+summary: Include search idle info to shard stats
+area: "Search"
+type: enhancement
+issues:
+ - 95727

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.stats/100_search_idle.yml

@@ -0,0 +1,28 @@
+---
+setup:
+ - do:
+ indices.create:
+ index: test
+ body:
+ settings:
+ number_of_shards: 2
+ number_of_replicas: 0
+ index:
+ search:
+ idle:
+ after: 60s
+
+---
+"Search idle stats":
+ - skip:
+ version: " - 8.8.99"
+ reason: "search idle stats added in 8.9.0"
+
+ - do:
+ indices.stats: { level: shards }
+
+ - is_false: indices.test.shards.0.0.search_idle
+ - gte: { indices.test.shards.0.0.search_idle_time: 0 }
+
+ - is_false: indices.test.shards.1.0.search_idle
+ - gte: { indices.test.shards.1.0.search_idle_time: 0 }

server/src/internalClusterTest/java/org/elasticsearch/index/shard/SearchIdleIT.java

@@ -9,9 +9,13 @@
 package org.elasticsearch.index.shard;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsRequest;
+import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.elasticsearch.action.admin.indices.stats.ShardStats;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.get.MultiGetRequest;
 import org.elasticsearch.action.index.IndexResponse;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.index.IndexService;
@@ -21,9 +25,11 @@
 import org.elasticsearch.xcontent.XContentType;
 
 import java.util.Arrays;
+import java.util.Locale;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Phaser;
 import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.IntToLongFunction;
 
@@ -127,6 +133,12 @@ public void onFailure(Exception e) {
  }
  indexingDone.await();
  t.join();
+ final IndicesStatsResponse statsResponse = client().admin().indices().stats(new IndicesStatsRequest()).actionGet();
+ for (ShardStats shardStats : statsResponse.getShards()) {
+ if (randomTimeValue != null && shardStats.isSearchIdle()) {
+ assertTrue(shardStats.getSearchIdleTime() >= randomTimeValue.millis());
+ }
+ }
  }
 
  public void testPendingRefreshWithIntervalChange() throws Exception {
@@ -167,6 +179,12 @@ public void testPendingRefreshWithIntervalChange() throws Exception {
  assertFalse(shard.hasRefreshPending());
  assertTrue(shard.isSearchIdle());
  assertHitCount(client().prepareSearch().get(), 3);
+ final IndicesStatsResponse statsResponse = client().admin().indices().stats(new IndicesStatsRequest()).actionGet();
+ for (ShardStats shardStats : statsResponse.getShards()) {
+ if (shardStats.isSearchIdle()) {
+ assertTrue(shardStats.getSearchIdleTime() >= TimeValue.ZERO.millis());
+ }
+ }
  }
 
  private void ensureNoPendingScheduledRefresh(ThreadPool threadPool) {
@@ -182,4 +200,27 @@ private void ensureNoPendingScheduledRefresh(ThreadPool threadPool) {
  barrier.arriveAndAwaitAdvance();
  }
 
+ public void testSearchIdleStats() throws InterruptedException {
+ int searchIdleAfter = randomIntBetween(2, 5);
+ final String indexName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
+ client().admin()
+ .indices()
+ .prepareCreate(indexName)
+ .setSettings(
+ Settings.builder()
+ .put(IndexSettings.INDEX_SEARCH_IDLE_AFTER.getKey(), searchIdleAfter + "s")
+ .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(2, 7))
+ )
+ .get();
+ waitUntil(
+ () -> Arrays.stream(client().admin().indices().prepareStats(indexName).get().getShards()).allMatch(ShardStats::isSearchIdle),
+ searchIdleAfter,
+ TimeUnit.SECONDS
+ );
+
+ final IndicesStatsResponse statsResponse = client().admin().indices().prepareStats(indexName).get();
+ assertTrue(Arrays.stream(statsResponse.getShards()).allMatch(ShardStats::isSearchIdle));
+ assertTrue(Arrays.stream(statsResponse.getShards()).allMatch(x -> x.getSearchIdleTime() >= searchIdleAfter));
+ }
+
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java

@@ -222,7 +222,9 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
  CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, SHARD_STATS_FLAGS),
  commitStats,
  seqNoStats,
- retentionLeaseStats
+ retentionLeaseStats,
+ indexShard.isSearchIdle(),
+ indexShard.searchIdleTime()
  )
  );
  }

server/src/main/java/org/elasticsearch/action/admin/indices/stats/ShardStats.java

@@ -41,6 +41,10 @@ public class ShardStats implements Writeable, ToXContentFragment {
  private final String statePath;
  private final boolean isCustomDataPath;
 
+ private final boolean isSearchIdle;
+
+ private final long searchIdleTime;
+
  public ShardStats(StreamInput in) throws IOException {
  shardRouting = new ShardRouting(in);
  commonStats = new CommonStats(in);
@@ -54,26 +58,13 @@ public ShardStats(StreamInput in) throws IOException {
  isCustomDataPath = in.readBoolean();
  seqNoStats = in.readOptionalWriteable(SeqNoStats::new);
  retentionLeaseStats = in.readOptionalWriteable(RetentionLeaseStats::new);
- }
-
- public ShardStats(
- final ShardRouting shardRouting,
- final ShardPath shardPath,
- final CommonStats commonStats,
- final CommitStats commitStats,
- final SeqNoStats seqNoStats,
- final RetentionLeaseStats retentionLeaseStats
- ) {
- this(
- shardRouting,
- commonStats,
- commitStats,
- seqNoStats,
- retentionLeaseStats,
- shardPath.getRootDataPath().toString(),
- shardPath.getRootStatePath().toString(),
- shardPath.isCustomDataPath()
- );
+ if (in.getTransportVersion().onOrAfter(TransportVersion.V_8_9_0)) {
+ isSearchIdle = in.readBoolean();
+ searchIdleTime = in.readVLong();
+ } else {
+ isSearchIdle = false;
+ searchIdleTime = 0;
+ }
  }
 
  public ShardStats(
@@ -84,7 +75,9 @@ public ShardStats(
  RetentionLeaseStats retentionLeaseStats,
  String dataPath,
  String statePath,
- boolean isCustomDataPath
+ boolean isCustomDataPath,
+ boolean isSearchIdle,
+ long searchIdleTime
  ) {
  this.shardRouting = shardRouting;
  this.commonStats = commonStats;
@@ -94,6 +87,32 @@ public ShardStats(
  this.dataPath = dataPath;
  this.statePath = statePath;
  this.isCustomDataPath = isCustomDataPath;
+ this.isSearchIdle = isSearchIdle;
+ this.searchIdleTime = searchIdleTime;
+ }
+
+ public ShardStats(
+ final ShardRouting shardRouting,
+ final ShardPath shardPath,
+ final CommonStats commonStats,
+ final CommitStats commitStats,
+ final SeqNoStats seqNoStats,
+ final RetentionLeaseStats retentionLeaseStats,
+ boolean isSearchIdle,
+ long searchIdleTime
+ ) {
+ this(
+ shardRouting,
+ commonStats,
+ commitStats,
+ seqNoStats,
+ retentionLeaseStats,
+ shardPath.getRootDataPath().toString(),
+ shardPath.getRootStatePath().toString(),
+ shardPath.isCustomDataPath(),
+ isSearchIdle,
+ searchIdleTime
+ );
  }
 
  @Override
@@ -108,12 +127,25 @@ public boolean equals(Object o) {
  && Objects.equals(commitStats, that.commitStats)
  && Objects.equals(commonStats, that.commonStats)
  && Objects.equals(seqNoStats, that.seqNoStats)
- && Objects.equals(retentionLeaseStats, that.retentionLeaseStats);
+ && Objects.equals(retentionLeaseStats, that.retentionLeaseStats)
+ && Objects.equals(isSearchIdle, that.isSearchIdle)
+ && Objects.equals(searchIdleTime, that.searchIdleTime);
  }
 
  @Override
  public int hashCode() {
- return Objects.hash(shardRouting, dataPath, statePath, isCustomDataPath, commitStats, commonStats, seqNoStats, retentionLeaseStats);
+ return Objects.hash(
+ shardRouting,
+ dataPath,
+ statePath,
+ isCustomDataPath,
+ commitStats,
+ commonStats,
+ seqNoStats,
+ retentionLeaseStats,
+ isSearchIdle,
+ searchIdleTime
+ );
  }
 
  /**
@@ -158,6 +190,14 @@ public boolean isCustomDataPath() {
  return isCustomDataPath;
  }
 
+ public boolean isSearchIdle() {
+ return isSearchIdle;
+ }
+
+ public long getSearchIdleTime() {
+ return searchIdleTime;
+ }
+
  @Override
  public void writeTo(StreamOutput out) throws IOException {
  shardRouting.writeTo(out);
@@ -172,6 +212,10 @@ public void writeTo(StreamOutput out) throws IOException {
  out.writeBoolean(isCustomDataPath);
  out.writeOptionalWriteable(seqNoStats);
  out.writeOptionalWriteable(retentionLeaseStats);
+ if (out.getTransportVersion().onOrAfter(TransportVersion.V_8_9_0)) {
+ out.writeBoolean(isSearchIdle);
+ out.writeVLong(searchIdleTime);
+ }
  }
 
  @Override
@@ -198,6 +242,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
  builder.field(Fields.DATA_PATH, dataPath);
  builder.field(Fields.IS_CUSTOM_DATA_PATH, isCustomDataPath);
  builder.endObject();
+ builder.field(Fields.SEARCH_IDLE, isSearchIdle);
+ builder.field(Fields.SEARCH_IDLE_TIME, searchIdleTime);
  return builder;
  }
 
@@ -211,6 +257,8 @@ static final class Fields {
  static final String PRIMARY = "primary";
  static final String NODE = "node";
  static final String RELOCATING_NODE = "relocating_node";
+ static final String SEARCH_IDLE = "search_idle";
+ static final String SEARCH_IDLE_TIME = "search_idle_time";
  }
 
 }

server/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java

@@ -135,7 +135,9 @@ protected void shardOperation(IndicesStatsRequest request, ShardRouting shardRou
  commonStats,
  commitStats,
  seqNoStats,
- retentionLeaseStats
+ retentionLeaseStats,
+ indexShard.isSearchIdle(),
+ indexShard.searchIdleTime()
  );
  });
  }

server/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -3783,6 +3783,10 @@ public final boolean isSearchIdle() {
  return (threadPool.relativeTimeInMillis() - lastSearcherAccess.get()) >= indexSettings.getSearchIdleAfter().getMillis();
  }
 
+ public long searchIdleTime() {
+ return threadPool.relativeTimeInMillis() - lastSearcherAccess.get();
+ }
+
  /**
  * Returns the last timestamp the searcher was accessed. This is a relative timestamp in milliseconds.
  */

server/src/main/java/org/elasticsearch/indices/IndicesService.java

@@ -537,7 +537,9 @@ IndexShardStats indexShardStats(final IndicesService indicesService, final Index
  CommonStats.getShardLevelStats(indicesService.getIndicesQueryCache(), indexShard, flags),
  commitStats,
  seqNoStats,
- retentionLeaseStats
+ retentionLeaseStats,
+ indexShard.isSearchIdle(),
+ indexShard.searchIdleTime()
  ) }
  );
  }

server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java

@@ -705,7 +705,7 @@ private static ShardStats createShardStats(ShardId shardId) {
  .resolve(shardRouting.shardId().getIndex().getUUID())
  .resolve(String.valueOf(shardRouting.shardId().id()));
  ShardPath shardPath = new ShardPath(false, path, path, shardRouting.shardId());
- return new ShardStats(shardRouting, shardPath, createShardLevelCommonStats(), null, null, null);
+ return new ShardStats(shardRouting, shardPath, createShardLevelCommonStats(), null, null, null, false, 0);
  }
 
  public static NodeStats createNodeStats() {

server/src/test/java/org/elasticsearch/action/admin/cluster/stats/VersionStatsTests.java

@@ -117,7 +117,9 @@ public void testCreation() {
  CommonStats.getShardLevelStats(null, indexShard, new CommonStatsFlags(CommonStatsFlags.Flag.Store)),
  null,
  null,
- null
+ null,
+ false,
+ 0
  );
  ClusterStatsNodeResponse nodeResponse = new ClusterStatsNodeResponse(
  TestDiscoveryNode.create("id"),

server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java

@@ -490,7 +490,7 @@ public static IndicesStatsResponse randomIndicesStatsResponse(final IndexMetadat
  stats.get = new GetStats();
  stats.flush = new FlushStats();
  stats.warmer = new WarmerStats();
- shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null));
+ shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null, false, 0));
  }
  }
  return IndicesStatsTests.newIndicesStatsResponse(

server/src/test/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsResponseTests.java

@@ -75,7 +75,7 @@ public void testGetIndices() {
  Path path = createTempDir().resolve("indices").resolve(index.getUUID()).resolve(String.valueOf(shardId));
  ShardPath shardPath = new ShardPath(false, path, path, shId);
  ShardRouting routing = createShardRouting(shId, (shardId == 0));
- shards.add(new ShardStats(routing, shardPath, null, null, null, null));
+ shards.add(new ShardStats(routing, shardPath, null, null, null, null, false, 0));
  AtomicLong primaryShardsCounter = expectedIndexToPrimaryShardsCount.computeIfAbsent(
  index.getName(),
  k -> new AtomicLong(0L)
@@ -124,7 +124,7 @@ public void testChunkedEncodingPerIndex() throws IOException {
  Path path = createTempDir().resolve("indices").resolve(shId.getIndex().getUUID()).resolve(String.valueOf(shId.id()));
  ShardPath shardPath = new ShardPath(false, path, path, shId);
  ShardRouting routing = createShardRouting(shId, (shId.id() == 0));
- stats.add(new ShardStats(routing, shardPath, new CommonStats(), null, null, null));
+ stats.add(new ShardStats(routing, shardPath, new CommonStats(), null, null, null, false, 0));
  }
  final IndicesStatsResponse indicesStatsResponse = new IndicesStatsResponse(
  stats.toArray(new ShardStats[0]),

server/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java

@@ -124,9 +124,18 @@ public void testFillShardLevelInfo() {
  CommonStats commonStats2 = new CommonStats();
  commonStats2.store = new StoreStats(1000, 999, 0L);
  ShardStats[] stats = new ShardStats[] {
- new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, test_0.shardId()), commonStats0, null, null, null),
- new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats1, null, null, null),
- new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats2, null, null, null) };
+ new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, test_0.shardId()), commonStats0, null, null, null, false, 0),
+ new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, test_1.shardId()), commonStats1, null, null, null, false, 0),
+ new ShardStats(
+ test_1,
+ new ShardPath(false, test1Path, test1Path, test_1.shardId()),
+ commonStats2,
+ null,
+ null,
+ null,
+ false,
+ 0
+ ) };
  Map<String, Long> shardSizes = new HashMap<>();
  Map<ShardId, Long> shardDataSetSizes = new HashMap<>();
  Map<ClusterInfo.NodeAndShard, String> routingToPath = new HashMap<>();

server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataStatsTests.java

@@ -114,6 +114,6 @@ private ShardStats createShardStats(
  .add(
  new IndexingStats.Stats(0, 0, 0, 0, 0, 0, 0, 0, false, 0, totalIndexingTimeSinceShardStartedInNanos, totalActiveTimeInNanos)
  );
- return new ShardStats(shardRouting, commonStats, null, null, null, null, null, false);
+ return new ShardStats(shardRouting, commonStats, null, null, null, null, null, false, false, 0);
  }
 }