Switch TDigestState to use HybridDigest by default

docs/changelog/96904.yaml

@@ -0,0 +1,18 @@
+pr: 96904
+summary: Switch TDigestState to use `HybridDigest` by default
+area: Aggregations
+type: breaking
+issues: []
+breaking:
+ title: Switch TDigestState to use `HybridDigest` by default
+ area: REST API
+ details: "The default implementation for TDigest in percentile calculations switches\
+ \ to a new internal implementation offering superior performance (2x-10x speedup),\
+ \ at a very small accuracy penalty for very large sample populations."
+ impact: "This change leads to generating slightly different results in percentile\
+ \ calculations. If the highest possible accuracy is desired, or it's crucial to\
+ \ produce exactly the same results as in previous versions, one can either set\
+ \ `execution_hint` to `high_accuracy` in the `tdigest` spec of a given percentile\
+ \ calculation, or set `search.aggs.tdigest_execution_hint` to `high_accuracy`\
+ \ in cluster settings to apply to all percentile queries."
+ notable: true

docs/reference/aggregations/metrics/boxplot-aggregation.asciidoc

@@ -55,9 +55,9 @@ The response will look like this:
  "load_time_boxplot": {
  "min": 0.0,
  "max": 990.0,
- "q1": 165.0,
+ "q1": 167.5,
  "q2": 445.0,
- "q3": 725.0,
+ "q3": 722.5,
  "lower": 0.0,
  "upper": 990.0
  }
@@ -114,9 +114,9 @@ GET latency/_search
  "load_time_boxplot": {
  "min": 0.0,
  "max": 99.0,
- "q1": 16.5,
+ "q1": 16.75,
  "q2": 44.5,
- "q3": 72.5,
+ "q3": 72.25,
  "lower": 0.0,
  "upper": 99.0
  }

docs/reference/aggregations/metrics/percentile-aggregation.asciidoc

@@ -54,19 +54,26 @@ percentiles: `[ 1, 5, 25, 50, 75, 95, 99 ]`. The response will look like this:
  "aggregations": {
  "load_time_outlier": {
  "values": {
- "1.0": 5.0,
- "5.0": 25.0,
- "25.0": 165.0,
+ "1.0": 10.0,
+ "5.0": 30.0,
+ "25.0": 170.0,
  "50.0": 445.0,
- "75.0": 725.0,
- "95.0": 945.0,
- "99.0": 985.0
+ "75.0": 720.0,
+ "95.0": 940.0,
+ "99.0": 980.0
  }
  }
  }
 }
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/"1.0": 10.0/"1.0": 9.9/]
+// TESTRESPONSE[s/"5.0": 30.0/"5.0": 29.5/]
+// TESTRESPONSE[s/"25.0": 170.0/"25.0": 167.5/]
+// TESTRESPONSE[s/"50.0": 445.0/"50.0": 445.0/]
+// TESTRESPONSE[s/"75.0": 720.0/"75.0": 722.5/]
+// TESTRESPONSE[s/"95.0": 940.0/"95.0": 940.5/]
+// TESTRESPONSE[s/"99.0": 980.0/"99.0": 980.1/]
 
 As you can see, the aggregation will return a calculated value for each percentile
 in the default range. If we assume response times are in milliseconds, it is
@@ -128,38 +135,45 @@ Response:
  "values": [
  {
  "key": 1.0,
- "value": 5.0
+ "value": 10.0
  },
  {
  "key": 5.0,
- "value": 25.0
+ "value": 30.0
  },
  {
  "key": 25.0,
- "value": 165.0
+ "value": 170.0
  },
  {
  "key": 50.0,
  "value": 445.0
  },
  {
  "key": 75.0,
- "value": 725.0
+ "value": 720.0
  },
  {
  "key": 95.0,
- "value": 945.0
+ "value": 940.0
  },
  {
  "key": 99.0,
- "value": 985.0
+ "value": 980.0
  }
  ]
  }
  }
 }
 --------------------------------------------------
 // TESTRESPONSE[s/\.\.\./"took": $body.took,"timed_out": false,"_shards": $body._shards,"hits": $body.hits,/]
+// TESTRESPONSE[s/"value": 10.0/"value": 9.9/]
+// TESTRESPONSE[s/"value": 30.0/"value": 29.5/]
+// TESTRESPONSE[s/"value": 170.0/"value": 167.5/]
+// TESTRESPONSE[s/"value": 445.0/"value": 445.0/]
+// TESTRESPONSE[s/"value": 720.0/"value": 722.5/]
+// TESTRESPONSE[s/"value": 940.0/"value": 940.5/]
+// TESTRESPONSE[s/"value": 980.0/"value": 980.1/]
 
 ==== Script
 
@@ -203,13 +217,13 @@ GET latency/_search
  "aggregations": {
  "load_time_outlier": {
  "values": {
- "1.0": 0.5,
- "5.0": 2.5,
- "25.0": 16.5,
+ "1.0": 0.99,
+ "5.0": 2.95,
+ "25.0": 16.75,
  "50.0": 44.5,
- "75.0": 72.5,
- "95.0": 94.5,
- "99.0": 98.5
+ "75.0": 72.25,
+ "95.0": 94.05,
+ "99.0": 98.01
  }
  }
  }

docs/reference/aggregations/pipeline/moving-percentiles-aggregation.asciidoc

@@ -97,7 +97,7 @@ And the following may be the response:
  "doc_count": 3,
  "the_percentile": {
  "values": {
- "1.0": 150.0,
+ "1.0": 151.0,
  "99.0": 200.0
  }
  }
@@ -108,13 +108,13 @@ And the following may be the response:
  "doc_count": 2,
  "the_percentile": {
  "values": {
- "1.0": 10.0,
- "99.0": 50.0
+ "1.0": 10.4,
+ "99.0": 49.6
  }
  },
  "the_movperc": {
  "values": {
- "1.0": 150.0,
+ "1.0": 151.0,
  "99.0": 200.0
  }
  }
@@ -125,13 +125,13 @@ And the following may be the response:
  "doc_count": 2,
  "the_percentile": {
  "values": {
- "1.0": 175.0,
- "99.0": 200.0
+ "1.0": 175.25,
+ "99.0": 199.75
  }
  },
  "the_movperc": {
  "values": {
- "1.0": 10.0,
+ "1.0": 11.6,
  "99.0": 200.0
  }
  }

modules/aggregations/build.gradle

@@ -43,7 +43,17 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task ->
  task.skipTest("search.aggregation/20_terms/numeric profiler", "The profiler results aren't backwards compatible.")
  task.skipTest("search.aggregation/210_top_hits_nested_metric/top_hits aggregation with sequence numbers", "#42809 the use nested path and filter sort throws an exception")
  task.skipTest("search.aggregation/370_doc_count_field/Test filters agg with doc_count", "Uses profiler for assertions which is not backwards compatible")
- task.skipTest("search.aggregation/420_percentile_ranks_tdigest_metric/filtered", "Uses t-digest library which is not backwards compatible")
+
+ // In 8.9.0, the default t-digest algorithm changed from AVL-tree-based to hybrid, combining a sorted array of samples with a merging
+ // implementation. This change leads to slight different percentile results, compared to previous versions.
+ task.skipTest("search.aggregation/180_percentiles_tdigest_metric/Basic test", "Hybrid t-digest produces different results.")
+ task.skipTest("search.aggregation/180_percentiles_tdigest_metric/Non-keyed test", "Hybrid t-digest produces different results.")
+ task.skipTest("search.aggregation/180_percentiles_tdigest_metric/Only aggs test", "Hybrid t-digest produces different results.")
+ task.skipTest("search.aggregation/180_percentiles_tdigest_metric/Explicit Percents test", "Hybrid t-digest produces different results.")
+ task.skipTest("search.aggregation/180_percentiles_tdigest_metric/Metadata test", "Hybrid t-digest produces different results.")
+ task.skipTest("search.aggregation/180_percentiles_tdigest_metric/Filtered test", "Hybrid t-digest produces different results.")
+ task.skipTest("search.aggregation/420_percentile_ranks_tdigest_metric/filtered", "Hybrid t-digest produces different results.")
+
  task.addAllowedWarningRegex("\\[types removal\\].*")
 }
 

modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/percentiles_tdigest_metric.yml

@@ -46,6 +46,9 @@ setup:
 
 ---
 "Basic test":
+ - skip:
+ version: " - 8.8.99"
+ reason: t-digest forked in 8.9.0
  - do:
  search:
  rest_total_hits_as_int: true
@@ -61,21 +64,21 @@ setup:
  - match: { hits.total: 4 }
  - length: { hits.hits: 4 }
 
- - match: { aggregations.percentiles_int.values.1\.0: 1.0 }
- - match: { aggregations.percentiles_int.values.5\.0: 1.0 }
- - match: { aggregations.percentiles_int.values.25\.0: 26.0 }
+ - match: { aggregations.percentiles_int.values.1\.0: 2.5 }
+ - match: { aggregations.percentiles_int.values.5\.0: 8.5 }
+ - match: { aggregations.percentiles_int.values.25\.0: 38.5 }
  - match: { aggregations.percentiles_int.values.50\.0: 76.0 }
- - match: { aggregations.percentiles_int.values.75\.0: 126.0 }
- - match: { aggregations.percentiles_int.values.95\.0: 151.0 }
- - match: { aggregations.percentiles_int.values.99\.0: 151.0 }
+ - match: { aggregations.percentiles_int.values.75\.0: 113.5 }
+ - match: { aggregations.percentiles_int.values.95\.0: 143.5 }
+ - match: { aggregations.percentiles_int.values.99\.0: 149.5 }
 
- - match: { aggregations.percentiles_double.values.1\.0: 1.0 }
- - match: { aggregations.percentiles_double.values.5\.0: 1.0 }
- - match: { aggregations.percentiles_double.values.25\.0: 26.0 }
+ - match: { aggregations.percentiles_double.values.1\.0: 2.5 }
+ - match: { aggregations.percentiles_double.values.5\.0: 8.5 }
+ - match: { aggregations.percentiles_double.values.25\.0: 38.5 }
  - match: { aggregations.percentiles_double.values.50\.0: 76.0 }
- - match: { aggregations.percentiles_double.values.75\.0: 126.0 }
- - match: { aggregations.percentiles_double.values.95\.0: 151.0 }
- - match: { aggregations.percentiles_double.values.99\.0: 151.0 }
+ - match: { aggregations.percentiles_double.values.75\.0: 113.5 }
+ - match: { aggregations.percentiles_double.values.95\.0: 143.5 }
+ - match: { aggregations.percentiles_double.values.99\.0: 149.5 }
 
  - do:
  search:
@@ -97,21 +100,21 @@ setup:
  - match: { hits.total: 4 }
  - length: { hits.hits: 4 }
 
- - match: { aggregations.percentiles_int.values.1\.0: 1.0 }
- - match: { aggregations.percentiles_int.values.5\.0: 1.0 }
- - match: { aggregations.percentiles_int.values.25\.0: 26.0 }
+ - match: { aggregations.percentiles_int.values.1\.0: 2.5 }
+ - match: { aggregations.percentiles_int.values.5\.0: 8.5 }
+ - match: { aggregations.percentiles_int.values.25\.0: 38.5 }
  - match: { aggregations.percentiles_int.values.50\.0: 76.0 }
- - match: { aggregations.percentiles_int.values.75\.0: 126.0 }
- - match: { aggregations.percentiles_int.values.95\.0: 151.0 }
- - match: { aggregations.percentiles_int.values.99\.0: 151.0 }
+ - match: { aggregations.percentiles_int.values.75\.0: 113.5 }
+ - match: { aggregations.percentiles_int.values.95\.0: 143.5 }
+ - match: { aggregations.percentiles_int.values.99\.0: 149.5 }
 
- - match: { aggregations.percentiles_double.values.1\.0: 1.0 }
- - match: { aggregations.percentiles_double.values.5\.0: 1.0 }
- - match: { aggregations.percentiles_double.values.25\.0: 26.0 }
+ - match: { aggregations.percentiles_double.values.1\.0: 2.5 }
+ - match: { aggregations.percentiles_double.values.5\.0: 8.5 }
+ - match: { aggregations.percentiles_double.values.25\.0: 38.5 }
  - match: { aggregations.percentiles_double.values.50\.0: 76.0 }
- - match: { aggregations.percentiles_double.values.75\.0: 126.0 }
- - match: { aggregations.percentiles_double.values.95\.0: 151.0 }
- - match: { aggregations.percentiles_double.values.99\.0: 151.0 }
+ - match: { aggregations.percentiles_double.values.75\.0: 113.5 }
+ - match: { aggregations.percentiles_double.values.95\.0: 143.5 }
+ - match: { aggregations.percentiles_double.values.99\.0: 149.5 }
 
 
 ---

server/src/main/java/org/elasticsearch/search/aggregations/metrics/MedianAbsoluteDeviationAggregator.java

@@ -92,7 +92,7 @@ public void collect(int doc, long bucket) throws IOException {
 
  TDigestState valueSketch = valueSketches.get(bucket);
  if (valueSketch == null) {
- valueSketch = TDigestState.create(compression);
+ valueSketch = TDigestState.create(compression, executionHint);
  valueSketches.set(bucket, valueSketch);
  }
 

server/src/main/java/org/elasticsearch/search/aggregations/metrics/TDigestState.java

@@ -38,7 +38,7 @@ protected enum Type {
  SORTING;
 
  static Type defaultValue() {
- return AVL_TREE;
+ return HYBRID;
  }
 
  static Type valueForHighAccuracy() {

server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestPercentilesAggregatorTests.java

@@ -83,8 +83,8 @@ public void testSomeMatchesSortedNumericDocValues() throws IOException {
  }, tdigest -> {
  assertEquals(7L, tdigest.state.size());
  assertEquals(7L, tdigest.state.centroidCount());
- assertEquals(4.5d, tdigest.percentile(75), 0.0d);
- assertEquals("4.5", tdigest.percentileAsString(75));
+ assertEquals(4.0d, tdigest.percentile(75), 0.0d);
+ assertEquals("4.0", tdigest.percentileAsString(75));
  assertEquals(2.0d, tdigest.percentile(50), 0.0d);
  assertEquals("2.0", tdigest.percentileAsString(50));
  assertEquals(1.0d, tdigest.percentile(22), 0.0d);
@@ -107,14 +107,14 @@ public void testSomeMatchesNumericDocValues() throws IOException {
  assertEquals(tdigest.state.centroidCount(), 7L);
  assertEquals(8.0d, tdigest.percentile(100), 0.0d);
  assertEquals("8.0", tdigest.percentileAsString(100));
- assertEquals(6.98d, tdigest.percentile(88), 0.0d);
- assertEquals("6.98", tdigest.percentileAsString(88));
+ assertEquals(4.0d, tdigest.percentile(75), 0.0d);
+ assertEquals("4.0", tdigest.percentileAsString(75));
  assertEquals(1.0d, tdigest.percentile(33), 0.0d);
  assertEquals("1.0", tdigest.percentileAsString(33));
  assertEquals(1.0d, tdigest.percentile(25), 0.0d);
  assertEquals("1.0", tdigest.percentileAsString(25));
- assertEquals(0.0d, tdigest.percentile(1), 0.0d);
- assertEquals("0.0", tdigest.percentileAsString(1));
+ assertEquals(0.06d, tdigest.percentile(1), 0.0d);
+ assertEquals("0.06", tdigest.percentileAsString(1));
  assertTrue(AggregationInspectionHelper.hasValue(tdigest));
  });
  }
@@ -135,7 +135,7 @@ public void testQueryFiltering() throws IOException {
  assertEquals(4L, tdigest.state.centroidCount());
  assertEquals(2.0d, tdigest.percentile(100), 0.0d);
  assertEquals(1.0d, tdigest.percentile(50), 0.0d);
- assertEquals(0.5d, tdigest.percentile(25), 0.0d);
+ assertEquals(0.75d, tdigest.percentile(25), 0.0d);
  assertTrue(AggregationInspectionHelper.hasValue(tdigest));
  });
 

server/src/test/java/org/elasticsearch/search/aggregations/metrics/TDigestStateTests.java

@@ -158,6 +158,8 @@ public void testFactoryMethods() {
 
  assertEquals(fast, anotherFast);
  assertEquals(accurate, anotherAccurate);
+ assertNotEquals(fast, accurate);
+ assertNotEquals(anotherFast, anotherAccurate);
  }
 
  private static TDigestState writeToAndReadFrom(TDigestState state, TransportVersion version) throws IOException {
@@ -195,6 +197,7 @@ public void testSerialization() throws IOException {
  assertEquals(serialized, state);
 
  TDigestState serializedBackwardsCompatible = writeToAndReadFrom(state, TransportVersion.V_8_500_000);
+ assertNotEquals(serializedBackwardsCompatible, state);
  assertEquals(serializedBackwardsCompatible, backwardsCompatible);
  }
 }

x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/aggregations/metrics/TDigestPreAggregatedPercentilesAggregatorTests.java

@@ -88,10 +88,10 @@ public void testSomeMatchesBinaryDocValues() throws IOException {
  }, hdr -> {
  // assertEquals(4L, hdr.state.getTotalCount());
  double approximation = 0.05d;
- assertEquals(15.0d, hdr.percentile(25), approximation);
+ assertEquals(17.5d, hdr.percentile(25), approximation);
  assertEquals(30.0d, hdr.percentile(50), approximation);
- assertEquals(50.0d, hdr.percentile(75), approximation);
- assertEquals(60.0d, hdr.percentile(99), approximation);
+ assertEquals(45.0d, hdr.percentile(75), approximation);
+ assertEquals(59.4d, hdr.percentile(99), approximation);
  assertTrue(AggregationInspectionHelper.hasValue(hdr));
  });
  }
@@ -105,9 +105,9 @@ public void testSomeMatchesMultiBinaryDocValues() throws IOException {
  }, hdr -> {
  // assertEquals(16L, hdr.state.getTotalCount());
  double approximation = 0.05d;
- assertEquals(15.0d, hdr.percentile(25), approximation);
+ assertEquals(17.5d, hdr.percentile(25), approximation);
  assertEquals(30.0d, hdr.percentile(50), approximation);
- assertEquals(50.0d, hdr.percentile(75), approximation);
+ assertEquals(45.0d, hdr.percentile(75), approximation);
  assertEquals(60.0d, hdr.percentile(99), approximation);
  assertTrue(AggregationInspectionHelper.hasValue(hdr));
  });