Aggregations: Added percentile rank aggregation

Percentile Rank Aggregation is the reverse of the Percetiles aggregation. It determines the percentile rank (the proportion of values less than a given value) of the provided array of values. Closes elastic#6386

Author: colings86

docs/reference/search/aggregations/metrics.asciidoc

@@ -16,6 +16,8 @@ include::metrics/valuecount-aggregation.asciidoc[]
 
 include::metrics/percentile-aggregation.asciidoc[]
 
+include::metrics/percentile-rank-aggregation.asciidoc[]
+
 include::metrics/cardinality-aggregation.asciidoc[]
 
 include::metrics/geobounds-aggregation.asciidoc[]

docs/reference/search/aggregations/metrics/percentile-aggregation.asciidoc

@@ -125,6 +125,7 @@ a script to convert them on-the-fly:
 script to generate values which percentiles are calculated on
 <2> Scripting supports parameterized input just like any other script
 
+[[search-aggregations-metrics-percentile-aggregation-approximation]]
 ==== Percentiles are (usually) approximate
 
 There are many different algorithms to calculate percentiles. The naive
@@ -161,6 +162,7 @@ for large number of values is that the law of large numbers makes the distributi
 values more and more uniform and the t-digest tree can do a better job at summarizing
 it. It would not be the case on more skewed distributions.
 
+[[search-aggregations-metrics-percentile-aggregation-compression]]
 ==== Compression
 
 Approximate algorithms must balance memory utilization with estimation accuracy.

docs/reference/search/aggregations/metrics/percentile-rank-aggregation.asciidoc

@@ -0,0 +1,95 @@
+[[search-aggregations-metrics-percentile-rank-aggregation]]
+=== Percentile Ranks Aggregation
+
+coming[1.3.0]
+
+A `multi-value` metrics aggregation that calculates one or more percentile ranks
+over numeric values extracted from the aggregated documents. These values
+can be extracted either from specific numeric fields in the documents, or
+be generated by a provided script.
+
+.Experimental!
+[IMPORTANT]
+=====
+This feature is marked as experimental, and may be subject to change in the
+future. If you use this feature, please let us know your experience with it!
+=====
+
+[NOTE]
+==================================================
+Please see <<search-aggregations-metrics-percentile-aggregation-approximation>> 
+and <<search-aggregations-metrics-percentile-aggregation-compression>> for advice 
+regarding approximation and memory use of the percentile ranks aggregation
+==================================================
+
+Percentile rank show the percentage of observed values which are below certain 
+value. For example, if a value is greater than or equal to 95% of the observed values
+it is said to be at the 95th percentile rank.
+
+Assume your data consists of website load times. You may have a service agreement that 
+95% of page loads completely within 15ms and 99% of page loads complete within 30ms.
+
+Let's look at a range of percentiles representing load time:
+
+[source,js]
+--------------------------------------------------
+{
+ "aggs" : {
+ "load_time_outlier" : {
+ "percentile_ranks" : {
+ "field" : "load_time" <1>
+ "values" : [15, 30]
+ }
+ }
+ }
+}
+--------------------------------------------------
+<1> The field `load_time` must be a numeric field
+
+The response will look like this:
+
+[source,js]
+--------------------------------------------------
+{
+ ...
+
+ "aggregations": {
+ "load_time_outlier": {
+ "values" : {
+ "15": 92,
+ "30": 100
+ }
+ }
+ }
+}
+--------------------------------------------------
+
+From this information you can determine you are hitting the 99% load time target but not quite 
+hitting the 95% load time target
+
+
+==== Script
+
+The percentile rank metric supports scripting. For example, if our load times
+are in milliseconds but we want to specify values in seconds, we could use
+a script to convert them on-the-fly:
+
+[source,js]
+--------------------------------------------------
+{
+ "aggs" : {
+ "load_time_outlier" : {
+ "percentile_ranks" : {
+ "values" : [3, 5],
+ "script" : "doc['load_time'].value / timeUnit", <1>
+ "params" : {
+ "timeUnit" : 1000 <2>
+ }
+ }
+ }
+ }
+}
+--------------------------------------------------
+<1> The `field` parameter is replaced with a `script` parameter, which uses the
+script to generate values which percentile ranks are calculated on
+<2> Scripting supports parameterized input just like any other script

src/main/java/org/elasticsearch/search/aggregations/AggregationBuilders.java

@@ -39,6 +39,7 @@
 import org.elasticsearch.search.aggregations.metrics.max.MaxBuilder;
 import org.elasticsearch.search.aggregations.metrics.min.MinBuilder;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesBuilder;
+import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksBuilder;
 import org.elasticsearch.search.aggregations.metrics.stats.StatsBuilder;
 import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsBuilder;
 import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder;
@@ -140,6 +141,10 @@ public static PercentilesBuilder percentiles(String name) {
  return new PercentilesBuilder(name);
  }
 
+ public static PercentileRanksBuilder percentileRanks(String name) {
+ return new PercentileRanksBuilder(name);
+ }
+
  public static CardinalityBuilder cardinality(String name) {
  return new CardinalityBuilder(name);
  }

src/main/java/org/elasticsearch/search/aggregations/AggregationModule.java

@@ -42,6 +42,7 @@
 import org.elasticsearch.search.aggregations.metrics.max.MaxParser;
 import org.elasticsearch.search.aggregations.metrics.min.MinParser;
 import org.elasticsearch.search.aggregations.metrics.percentiles.PercentilesParser;
+import org.elasticsearch.search.aggregations.metrics.percentiles.PercentileRanksParser;
 import org.elasticsearch.search.aggregations.metrics.stats.StatsParser;
 import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsParser;
 import org.elasticsearch.search.aggregations.metrics.sum.SumParser;
@@ -65,6 +66,7 @@ public AggregationModule() {
  parsers.add(ExtendedStatsParser.class);
  parsers.add(ValueCountParser.class);
  parsers.add(PercentilesParser.class);
+ parsers.add(PercentileRanksParser.class);
  parsers.add(CardinalityParser.class);
 
  parsers.add(GlobalParser.class);

src/main/java/org/elasticsearch/search/aggregations/TransportAggregationModule.java

@@ -45,6 +45,7 @@
 import org.elasticsearch.search.aggregations.metrics.max.InternalMax;
 import org.elasticsearch.search.aggregations.metrics.min.InternalMin;
 import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentiles;
+import org.elasticsearch.search.aggregations.metrics.percentiles.InternalPercentileRanks;
 import org.elasticsearch.search.aggregations.metrics.stats.InternalStats;
 import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats;
 import org.elasticsearch.search.aggregations.metrics.sum.InternalSum;
@@ -67,6 +68,7 @@ protected void configure() {
  InternalExtendedStats.registerStreams();
  InternalValueCount.registerStreams();
  InternalPercentiles.registerStreams();
+ InternalPercentileRanks.registerStreams();
  InternalCardinality.registerStreams();
 
  // buckets

src/main/java/org/elasticsearch/search/aggregations/metrics/percentiles/AbstractInternalPercentiles.java

@@ -0,0 +1,137 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.search.aggregations.metrics.percentiles;
+
+import org.elasticsearch.ElasticsearchIllegalArgumentException;
+import org.elasticsearch.Version;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.metrics.InternalNumericMetricsAggregation;
+import org.elasticsearch.search.aggregations.metrics.percentiles.tdigest.TDigestState;
+import org.elasticsearch.search.aggregations.support.format.ValueFormatterStreams;
+
+import java.io.IOException;
+import java.util.List;
+
+abstract class AbstractInternalPercentiles extends InternalNumericMetricsAggregation.MultiValue {
+
+ protected double[] keys;
+ protected TDigestState state;
+ private boolean keyed;
+
+ AbstractInternalPercentiles() {} // for serialization
+
+ public AbstractInternalPercentiles(String name, double[] keys, TDigestState state, boolean keyed) {
+ super(name);
+ this.keys = keys;
+ this.state = state;
+ this.keyed = keyed;
+ }
+
+ @Override
+ public double value(String name) {
+ return value(Double.parseDouble(name));
+ }
+ 
+ public abstract double value(double key);
+
+ @Override
+ public AbstractInternalPercentiles reduce(ReduceContext reduceContext) {
+ List<InternalAggregation> aggregations = reduceContext.aggregations();
+ TDigestState merged = null;
+ for (InternalAggregation aggregation : aggregations) {
+ final AbstractInternalPercentiles percentiles = (AbstractInternalPercentiles) aggregation;
+ if (merged == null) {
+ merged = new TDigestState(percentiles.state.compression());
+ }
+ merged.add(percentiles.state);
+ }
+ return createReduced(getName(), keys, merged, keyed);
+ }
+
+ protected abstract AbstractInternalPercentiles createReduced(String name, double[] keys, TDigestState merged, boolean keyed);
+
+ @Override
+ public void readFrom(StreamInput in) throws IOException {
+ name = in.readString();
+ valueFormatter = ValueFormatterStreams.readOptional(in);
+ if (in.getVersion().before(Version.V_1_2_0)) {
+ final byte id = in.readByte();
+ if (id != 0) {
+ throw new ElasticsearchIllegalArgumentException("Unexpected percentiles aggregator id [" + id + "]");
+ }
+ }
+ keys = new double[in.readInt()];
+ for (int i = 0; i < keys.length; ++i) {
+ keys[i] = in.readDouble();
+ }
+ state = TDigestState.read(in);
+ keyed = in.readBoolean();
+ }
+
+ @Override
+ public void writeTo(StreamOutput out) throws IOException {
+ out.writeString(name);
+ ValueFormatterStreams.writeOptional(valueFormatter, out);
+ if (out.getVersion().before(Version.V_1_2_0)) {
+ out.writeByte((byte) 0);
+ }
+ out.writeInt(keys.length);
+ for (int i = 0 ; i < keys.length; ++i) {
+ out.writeDouble(keys[i]);
+ }
+ TDigestState.write(state, out);
+ out.writeBoolean(keyed);
+ }
+
+ @Override
+ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+ builder.startObject(name);
+ if (keyed) {
+ builder.startObject(CommonFields.VALUES);
+ for(int i = 0; i < keys.length; ++i) {
+ String key = String.valueOf(keys[i]);
+ double value = value(keys[i]);
+ builder.field(key, value);
+ if (valueFormatter != null) {
+ builder.field(key + "_as_string", valueFormatter.format(value));
+ }
+ }
+ builder.endObject();
+ } else {
+ builder.startArray(CommonFields.VALUES);
+ for (int i = 0; i < keys.length; i++) {
+ double value = value(keys[i]);
+ builder.startObject();
+ builder.field(CommonFields.KEY, keys[i]);
+ builder.field(CommonFields.VALUE, value);
+ if (valueFormatter != null) {
+ builder.field(CommonFields.VALUE_AS_STRING, valueFormatter.format(value));
+ }
+ builder.endObject();
+ }
+ builder.endArray();
+ }
+ builder.endObject();
+ return builder;
+ }
+}