Linear retriever top level option for normalizer

docs/changelog/129693.yaml

@@ -0,0 +1,5 @@
+pr: 129693
+summary: Add top level normalizer for linear retriever
+area: Search
+type: enhancement
+issues: []

docs/reference/elasticsearch/rest-apis/retrievers/linear-retriever.md

@@ -31,9 +31,12 @@ Combining `query` and `retrievers` is not supported.
 `normalizer` {applies_to}`stack: ga 9.1`
 : (Optional, String)
 
- The normalizer to use when using the [multi-field query format](../retrievers.md#multi-field-query-format).
+ The normalizer to use when combining results.
  See [normalizers](#linear-retriever-normalizers) for supported values.
  Required when `query` is specified.
+
+ When used with the [multi-field query format](../retrievers.md#multi-field-query-format) (`query` parameter), normalizes scores per [field grouping](../retrievers.md#multi-field-field-grouping).
+ When used with `retrievers`, serves as the default normalizer for any sub-retriever that doesn't specify its own normalizer. Per-retriever normalizers always take precedence over the top-level normalizer.
 
  ::::{warning}
  Avoid using `none` as that will disable normalization and may bias the result set towards lexical matches.
@@ -74,9 +77,9 @@ Each entry in the `retrievers` array specifies the following parameters:
 `normalizer`
 : (Optional, String)
 
- Specifies how the retriever’s score will be normalized before applying the specified `weight`.
+ Specifies how the retriever's score will be normalized before applying the specified `weight`.
  See [normalizers](#linear-retriever-normalizers) for supported values.
- Defaults to `none`.
+ If not specified, uses the top-level `normalizer` or defaults to `none` if no top-level normalizer is set.
 
 See also [this hybrid search example](retrievers-examples.md#retrievers-examples-linear-retriever) using a linear retriever on how to independently configure and apply normalizers to retrievers.
 
@@ -94,7 +97,7 @@ The `linear` retriever supports the following normalizers:
 
 ## Example
 
-This example of a hybrid search weights KNN results five times more heavily than BM25 results in the final ranking.
+This example of a hybrid search weights KNN results five times more heavily than BM25 results in the final ranking, with a top-level normalizer applied to all retrievers.
 
 ```console
 GET my_index/_search
@@ -105,23 +108,33 @@ GET my_index/_search
  {
  "retriever": {
  "knn": {
- ...
+ "field": "title_vector",
+ "query_vector": [0.1, 0.2, 0.3],
+ "k": 10,
+ "num_candidates": 100
  }
  },
  "weight": 5 # KNN query weighted 5x
  },
  {
  "retriever": {
  "standard": {
- ...
+ "query": {
+ "match": {
+ "title": "elasticsearch"
+ }
+ }
  }
  },
  "weight": 1.5 # BM25 query weighted 1.5x
  }
- ]
+ ],
+ "normalizer": "minmax"
  }
  }
 }
 ```
 
+In this example, the `minmax` normalizer is applied to both the kNN retriever and the standard retriever. The top-level normalizer serves as a default that can be overridden by individual sub-retrievers. When using the multi-field query format, the top-level normalizer is applied to all generated inner retrievers.
+
 See also [this hybrid search example](retrievers-examples.md#retrievers-examples-linear-retriever).

docs/reference/elasticsearch/rest-apis/retrievers/retrievers-examples.md

@@ -247,7 +247,7 @@ GET /retrievers_example/_search
  }
  },
  "weight": 1.5,
- "normalizer": "minmax"
+ "normalizer": "l2_norm"
  }
  ],
  "rank_window_size": 10

x-pack/plugin/rank-rrf/src/internalClusterTest/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverIT.java

@@ -835,4 +835,43 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
  );
  assertThat(numAsyncCalls.get(), equalTo(4));
  }
+
+ public void testMixedNormalizerInheritance() throws IOException {
+ client().prepareIndex(INDEX)
+ .setId("1")
+ .setSource("field1", "elasticsearch search", "field2", "database technology", "score", 10)
+ .get();
+ client().prepareIndex(INDEX).setId("2").setSource("field1", "lucene engine", "field2", "search technology", "score", 5).get();
+ client().prepareIndex(INDEX)
+ .setId("3")
+ .setSource("field1", "information retrieval", "field2", "database search", "score", 15)
+ .get();
+ refresh(INDEX);
+
+ LinearRetrieverBuilder linearRetriever = new LinearRetrieverBuilder(
+ List.of(
+ CompoundRetrieverBuilder.RetrieverSource.from(
+ new StandardRetrieverBuilder(QueryBuilders.matchQuery("field1", "elasticsearch"))
+ ),
+ CompoundRetrieverBuilder.RetrieverSource.from(
+ new StandardRetrieverBuilder(QueryBuilders.matchQuery("field2", "technology"))
+ ),
+ CompoundRetrieverBuilder.RetrieverSource.from(new StandardRetrieverBuilder(QueryBuilders.matchQuery("field1", "search")))
+ ),
+ null,
+ null,
+ MinMaxScoreNormalizer.INSTANCE,
+ 10,
+ new float[] { 1.0f, 1.0f, 1.0f },
+ new ScoreNormalizer[] { null, L2ScoreNormalizer.INSTANCE, null }
+ );
+
+ assertThat(linearRetriever.getNormalizers()[0], equalTo(MinMaxScoreNormalizer.INSTANCE));
+ assertThat(linearRetriever.getNormalizers()[1], equalTo(L2ScoreNormalizer.INSTANCE));
+ assertThat(linearRetriever.getNormalizers()[2], equalTo(MinMaxScoreNormalizer.INSTANCE));
+
+ assertResponse(client().prepareSearch(INDEX).setSource(new SearchSourceBuilder().retriever(linearRetriever)), searchResponse -> {
+ assertThat(searchResponse.getHits().getTotalHits().value() > 0L, is(true));
+ });
+ }
 }

x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverBuilder.java

@@ -43,6 +43,7 @@
 import static org.elasticsearch.action.ValidateActions.addValidationError;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
 import static org.elasticsearch.xpack.rank.RankRRFFeatures.LINEAR_RETRIEVER_SUPPORTED;
+import static org.elasticsearch.xpack.rank.linear.LinearRetrieverComponent.DEFAULT_NORMALIZER;
 import static org.elasticsearch.xpack.rank.linear.LinearRetrieverComponent.DEFAULT_WEIGHT;
 
 /**
@@ -74,6 +75,16 @@ public final class LinearRetrieverBuilder extends CompoundRetrieverBuilder<Linea
  private final String query;
  private final ScoreNormalizer normalizer;
 
+ private static ScoreNormalizer resolveNormalizer(ScoreNormalizer componentNormalizer, ScoreNormalizer topLevelNormalizer) {
+ if (componentNormalizer != null) {
+ return componentNormalizer;
+ }
+ if (topLevelNormalizer != null) {
+ return topLevelNormalizer;
+ }
+ return DEFAULT_NORMALIZER;
+ }
+
  @SuppressWarnings("unchecked")
  static final ConstructingObjectParser<LinearRetrieverBuilder, RetrieverParserContext> PARSER = new ConstructingObjectParser<>(
  NAME,
@@ -92,7 +103,7 @@ public final class LinearRetrieverBuilder extends CompoundRetrieverBuilder<Linea
  for (LinearRetrieverComponent component : retrieverComponents) {
  innerRetrievers.add(RetrieverSource.from(component.retriever));
  weights[index] = component.weight;
- normalizers[index] = component.normalizer;
+ normalizers[index] = resolveNormalizer(component.normalizer, normalizer);
  index++;
  }
  return new LinearRetrieverBuilder(innerRetrievers, fields, query, normalizer, rankWindowSize, weights, normalizers);
@@ -118,7 +129,7 @@ private static float[] getDefaultWeight(List<RetrieverSource> innerRetrievers) {
  private static ScoreNormalizer[] getDefaultNormalizers(List<RetrieverSource> innerRetrievers) {
  int size = innerRetrievers != null ? innerRetrievers.size() : 0;
  ScoreNormalizer[] normalizers = new ScoreNormalizer[size];
- Arrays.fill(normalizers, IdentityScoreNormalizer.INSTANCE);
+ Arrays.fill(normalizers, DEFAULT_NORMALIZER);
  return normalizers;
  }
 
@@ -167,7 +178,10 @@ public LinearRetrieverBuilder(
  this.query = query;
  this.normalizer = normalizer;
  this.weights = weights;
- this.normalizers = normalizers;
+ this.normalizers = new ScoreNormalizer[normalizers.length];
+ for (int i = 0; i < normalizers.length; i++) {
+ this.normalizers[i] = resolveNormalizer(normalizers[i], normalizer);
+ }
  }
 
  public LinearRetrieverBuilder(
@@ -221,19 +235,7 @@ public ActionRequestValidationException validate(
  ),
  validationException
  );
- } else if (innerRetrievers.isEmpty() == false && normalizer != null) {
- validationException = addValidationError(
- String.format(
- Locale.ROOT,
- "[%s] [%s] cannot be provided when [%s] is specified",
- getName(),
- NORMALIZER_FIELD.getPreferredName(),
- RETRIEVERS_FIELD.getPreferredName()
- ),
- validationException
- );
  }
-
  return validationException;
  }
 
@@ -339,7 +341,7 @@ protected RetrieverBuilder doRewrite(QueryRewriteContext ctx) {
  for (var weightedRetriever : r) {
  retrievers.add(weightedRetriever.retrieverSource());
  weights[index] = weightedRetriever.weight();
- normalizers[index] = normalizer;
+ normalizers[index] = resolveNormalizer(null, normalizer);
  index++;
  }
 
@@ -357,7 +359,7 @@ protected RetrieverBuilder doRewrite(QueryRewriteContext ctx) {
  Arrays.fill(weights, DEFAULT_WEIGHT);
 
  ScoreNormalizer[] normalizers = new ScoreNormalizer[fieldsInnerRetrievers.size()];
- Arrays.fill(normalizers, normalizer);
+ Arrays.fill(normalizers, resolveNormalizer(null, normalizer));
 
  // TODO: This is a incomplete solution as it does not address other incomplete copy issues
  // (such as dropping the retriever name and min score)

x-pack/plugin/rank-rrf/src/main/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverComponent.java

@@ -38,7 +38,7 @@ public LinearRetrieverComponent(RetrieverBuilder retrieverBuilder, Float weight,
  assert retrieverBuilder != null;
  this.retriever = retrieverBuilder;
  this.weight = weight == null ? DEFAULT_WEIGHT : weight;
- this.normalizer = normalizer == null ? DEFAULT_NORMALIZER : normalizer;
+ this.normalizer = normalizer; // Don't default to identity, allow null for top-level fallback
  if (this.weight < 0) {
  throw new IllegalArgumentException("[weight] must be non-negative");
  }

x-pack/plugin/rank-rrf/src/test/java/org/elasticsearch/xpack/rank/linear/LinearRetrieverBuilderParsingTests.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
+import org.elasticsearch.xcontent.XContentType;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -27,6 +28,7 @@
 import java.util.List;
 
 import static java.util.Collections.emptyList;
+import static org.hamcrest.Matchers.instanceOf;
 
 public class LinearRetrieverBuilderParsingTests extends AbstractXContentTestCase<LinearRetrieverBuilder> {
  private static List<NamedXContentRegistry.Entry> xContentRegistryEntries;
@@ -67,7 +69,9 @@ protected LinearRetrieverBuilder createTestInstance() {
  new CompoundRetrieverBuilder.RetrieverSource(TestRetrieverBuilder.createRandomTestRetrieverBuilder(), null)
  );
  weights[i] = randomFloat();
- normalizers[i] = randomScoreNormalizer();
+ normalizers[i] = randomFrom(
+ new ScoreNormalizer[] { null, MinMaxScoreNormalizer.INSTANCE, L2ScoreNormalizer.INSTANCE, IdentityScoreNormalizer.INSTANCE }
+ );
  }
 
  return new LinearRetrieverBuilder(innerRetrievers, fields, query, normalizer, rankWindowSize, weights, normalizers);
@@ -108,10 +112,51 @@ protected NamedXContentRegistry xContentRegistry() {
  }
 
  private static ScoreNormalizer randomScoreNormalizer() {
- if (randomBoolean()) {
- return MinMaxScoreNormalizer.INSTANCE;
- } else {
- return IdentityScoreNormalizer.INSTANCE;
+ int random = randomInt(2);
+ return switch (random) {
+ case 0 -> MinMaxScoreNormalizer.INSTANCE;
+ case 1 -> L2ScoreNormalizer.INSTANCE;
+ default -> IdentityScoreNormalizer.INSTANCE;
+ };
+ }
+
+ public void testTopLevelNormalizer() throws IOException {
+ String json = """
+ {
+ "linear": {
+ "retrievers": [
+ {
+ "retriever": {
+ "test": {
+ "value": "test1"
+ }
+ },
+ "weight": 1.0,
+ "normalizer": "none"
+ },
+ {
+ "retriever": {
+ "test": {
+ "value": "test2"
+ }
+ },
+ "weight": 1.0,
+ "normalizer": "none"
+ }
+ ],
+ "normalizer": "minmax"
+ }
+ }""";
+
+ try (XContentParser parser = createParser(XContentType.JSON.xContent(), json)) {
+ LinearRetrieverBuilder builder = doParseInstance(parser);
+ // Test that the top-level normalizer is properly applied - the individual
+ // Per-retriever 'none' should override top-level 'minmax'
+ ScoreNormalizer[] normalizers = builder.getNormalizers();
+ assertEquals(2, normalizers.length);
+ for (ScoreNormalizer normalizer : normalizers) {
+ assertThat(normalizer, instanceOf(IdentityScoreNormalizer.class));
+ }
  }
  }
 }