Store high-cardinality keyword fields in binary doc values (#138548)

This PR adds a mapping parameter to keyword fields doc_values.cardinality. When this parameter is set to low (the default), keyword fields will use sorted set doc values as normal. However, when this parameter is set to high, keyword fields will instead use binary doc values. This is an optimization to remove the overhead of looking up keyword values by ordinal when the keyword field has high-cardinality.

Author: jordan-powers

modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.index.mapper.TestDocumentParserContext;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.script.field.BinaryDocValuesField;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.test.ESTestCase;
@@ -88,7 +89,7 @@ public void testStoringQueryBuilders() throws IOException {
  when(searchExecutionContext.getWriteableRegistry()).thenReturn(writableRegistry());
  when(searchExecutionContext.getParserConfig()).thenReturn(parserConfig());
  when(searchExecutionContext.getForField(fieldMapper.fieldType(), fielddataOperation)).thenReturn(
- new BytesBinaryIndexFieldData(fieldMapper.fullPath(), CoreValuesSourceType.KEYWORD)
+ new BytesBinaryIndexFieldData(fieldMapper.fullPath(), CoreValuesSourceType.KEYWORD, BinaryDocValuesField::new)
  );
  when(searchExecutionContext.getFieldType(Mockito.anyString())).thenAnswer(invocation -> {
  final String fieldName = (String) invocation.getArguments()[0];

qa/ccs-common-rest/build.gradle

@@ -32,15 +32,21 @@ dependencies {
 }
 
 tasks.named("yamlRestTest") {
- systemProperty 'tests.rest.blacklist',
- [
- 'search/150_rewrite_on_coordinator/Ensure that we fetch the document only once', // terms lookup query with index
- 'search/170_terms_query/Terms Query with No.of terms exceeding index.max_terms_count should FAIL', // terms lookup query with index
- 'search.aggregation/220_filters_bucket/cache busting', // node_selector?
- 'search.aggregation/220_filters_bucket/cache hits', // node_selector?
- 'search.aggregation/50_filter/Standard queries get cached',
- 'search.aggregation/50_filter/Terms lookup gets cached', // terms lookup by "index" doesn't seem to work correctly
- 'search.aggregation/70_adjacency_matrix/Terms lookup' // terms lookup by "index" doesn't seem to work correctly
- ].join(',')
+ ArrayList<String> blacklist = [
+ 'search/150_rewrite_on_coordinator/Ensure that we fetch the document only once', // terms lookup query with index
+ 'search/170_terms_query/Terms Query with No.of terms exceeding index.max_terms_count should FAIL', // terms lookup query with index
+ 'search.aggregation/220_filters_bucket/cache busting', // node_selector?
+ 'search.aggregation/220_filters_bucket/cache hits', // node_selector?
+ 'search.aggregation/50_filter/Standard queries get cached',
+ 'search.aggregation/50_filter/Terms lookup gets cached', // terms lookup by "index" doesn't seem to work correctly
+ 'search.aggregation/70_adjacency_matrix/Terms lookup' // terms lookup by "index" doesn't seem to work correctly
+ ]
+ if (buildParams.snapshotBuild == false) {
+ blacklist += [
+ // doc_values.cardinality option not available in snapshots
+ "search/395_binary_doc_values_search/*"
+ ]
+ }
+ systemProperty 'tests.rest.blacklist', blacklist.join(',')
 }
 

qa/smoke-test-multinode/build.gradle

@@ -24,9 +24,16 @@ dependencies {
 }
 
 tasks.named("yamlRestTest").configure {
- systemProperty 'tests.rest.blacklist', [
+ ArrayList<String> blacklist = [
  'cat.templates/10_basic/No templates',
  'cat.templates/10_basic/Sort templates',
  'cat.templates/10_basic/Multiple template',
- ].join(',')
+ ]
+ if (buildParams.snapshotBuild == false) {
+ blacklist += [
+ // doc_values.cardinality option not available in snapshots
+ "search/395_binary_doc_values_search/*"
+ ]
+ }
+ systemProperty 'tests.rest.blacklist', blacklist.join(',')
 }

rest-api-spec/build.gradle

@@ -54,6 +54,15 @@ tasks.named("precommit").configure {
  dependsOn 'enforceYamlTestConvention'
 }
 
+tasks.named("yamlRestTest") {
+ if (buildParams.snapshotBuild == false) {
+ // doc_values.cardinality option not available in snapshots
+ systemProperty 'tests.rest.blacklist', [
+ "search/395_binary_doc_values_search/*"
+ ].join(',')
+ }
+}
+
 tasks.named("yamlRestCompatTestTransform").configure ({ task ->
  task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling")
  task.replaceValueInMatch("profile.shards.0.dfs.knn.0.query.0.description", "DocAndScoreQuery[0,...][0.009673266,...],0.009673266", "dfs knn vector profiling with vector_operations_count")

rest-api-spec/src/yamlRestTest/java/org/elasticsearch/test/rest/ClientYamlTestSuiteIT.java

@@ -38,6 +38,7 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
  .feature(FeatureFlag.DOC_VALUES_SKIPPER)
  .feature(FeatureFlag.SYNTHETIC_VECTORS)
  .feature(FeatureFlag.RANDOM_SAMPLING)
+ .feature(FeatureFlag.EXTENDED_DOC_VALUES_PARAMS)
  .build();
 
  public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/395_binary_doc_values_search.yml

@@ -0,0 +1,115 @@
+---
+setup:
+ - requires:
+ cluster_features: ["mapper.keyword.store_high_cardinality_in_binary_doc_values"]
+ reason: "testing binary doc values search"
+
+ - do:
+ indices.create:
+ index: test
+ body:
+ mappings:
+ dynamic: false
+ properties:
+ keyword:
+ type: keyword
+ index: false
+ doc_values:
+ cardinality: high
+
+ - do:
+ index:
+ index: test
+ id: "1"
+ body:
+ keyword: "key1"
+
+ - do:
+ index:
+ index: test
+ id: "2"
+ body:
+ keyword: "key2"
+ - do:
+ indices.refresh: {}
+
+---
+"Test match query on keyword field where only binary doc values are enabled":
+
+ - do:
+ search:
+ index: test
+ body: { query: { match: { keyword: { query: "key1" } } } }
+ - length: { hits.hits: 1 }
+
+---
+"Test terms query on keyword field where only binary doc values are enabled":
+
+ - do:
+ search:
+ index: test
+ body: { query: { terms: { keyword: [ "key1", "key2" ] } } }
+ - length: { hits.hits: 2 }
+
+---
+"Test range query on keyword field where only binary doc values are enabled":
+
+ - do:
+ search:
+ index: test
+ body: { query: { range: { keyword: { gte: "key1" } } } }
+ - length: { hits.hits: 2 }
+
+---
+"Test fuzzy query on keyword field where only binary doc values are enabled":
+
+ - do:
+ search:
+ index: test
+ body: { query: { fuzzy: { keyword: { value: "kay1", fuzziness: 1 } } } }
+ - length: { hits.hits: 1 }
+
+---
+"Test prefix query on keyword field where only binary doc values are enabled":
+
+ - do:
+ search:
+ index: test
+ body: { query: { prefix: { keyword: { value: "key" } } } }
+ - length: { hits.hits: 2 }
+
+---
+"Test case insensitive term query on keyword field where only binary doc values are enabled":
+
+ - do:
+ search:
+ index: test
+ body: { query: { term: { keyword: { value: "KeY1", case_insensitive: true } } } }
+ - length: { hits.hits: 1 }
+
+---
+"Test wildcard query on keyword field where only binary doc values are enabled":
+
+ - do:
+ search:
+ index: test
+ body: { query: { wildcard: { keyword: { value: "k*1" } } } }
+ - length: { hits.hits: 1 }
+
+---
+"Test case insensitive wildcard query on keyword field where only binary doc values are enabled":
+
+ - do:
+ search:
+ index: test
+ body: { query: { wildcard: { keyword: { value: "K*1", case_insensitive: true } } } }
+ - length: { hits.hits: 1 }
+
+---
+"Test regexp query on keyword field where only binary doc values are enabled":
+
+ - do:
+ search:
+ index: test
+ body: { query: { regexp: { keyword: { value: "k.*1" } } } }
+ - length: { hits.hits: 1 }

server/src/main/java/org/elasticsearch/index/fielddata/MultiValuedSortedBinaryDocValues.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.index.fielddata;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.common.io.stream.ByteArrayStreamInput;
+
+import java.io.IOException;
+
+/**
+ * Wrapper around {@link BinaryDocValues} to decode the typical multivalued encoding used by
+ * {@link org.elasticsearch.index.mapper.BinaryFieldMapper.CustomBinaryDocValuesField}.
+ */
+public class MultiValuedSortedBinaryDocValues extends SortedBinaryDocValues {
+
+ BinaryDocValues values;
+ int count;
+
+ // the binary doc values for a document are all encoded in a single binary array, which this stream knows how to read
+ // the doc values in the array take the form of [doc value count][length of value 1][value 1][length of value 2][value 2]...
+ final ByteArrayStreamInput in = new ByteArrayStreamInput();
+ final BytesRef scratch = new BytesRef();
+
+ public MultiValuedSortedBinaryDocValues(BinaryDocValues values) {
+ this.values = values;
+ }
+
+ @Override
+ public boolean advanceExact(int doc) throws IOException {
+ if (values.advanceExact(doc)) {
+ final BytesRef bytes = values.binaryValue();
+ assert bytes.length > 0;
+ in.reset(bytes.bytes, bytes.offset, bytes.length);
+ count = in.readVInt();
+ scratch.bytes = bytes.bytes;
+ return true;
+ } else {
+ count = 0;
+ return false;
+ }
+ }
+
+ @Override
+ public int docValueCount() {
+ return count;
+ }
+
+ @Override
+ public BytesRef nextValue() throws IOException {
+ scratch.length = in.readVInt();
+ scratch.offset = in.getPosition();
+ in.setPosition(scratch.offset + scratch.length);
+ return scratch;
+ }
+}