Script: fields API for flattened mapped type (#82590)

* Script: fields API for flattened mapped type The flattened field type exposes all leaf values as keyword doc values. Additionally, specific keys are available via object dot notation. For example: ``` { "flat": { "abc": "bar", "def": "foo", "hij": { "lmn": "pqr", "stu": 123 } } } field('flat').get('default') // returns 123 field('flat.abc').get('default') // returns bar ``` API: * `iterator()` * `get(String default)` * `get(String default, int index)` Refs: #79105

Author: stu-elastic

docs/changelog/82590.yaml

@@ -0,0 +1,5 @@
+pr: 82590
+summary: "Script: fields API for flattened mapped type"
+area: Infra/Scripting
+type: enhancement
+issues: []

modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.fields.txt

@@ -92,11 +92,19 @@ class org.elasticsearch.script.field.DateNanosDocValuesField @dynamic_type {
  ZonedDateTime get(int, ZonedDateTime)
 }
 
-class org.elasticsearch.script.field.KeywordDocValuesField @dynamic_type {
+class org.elasticsearch.script.field.AbstractKeywordDocValuesField @dynamic_type {
  String get(String)
  String get(int, String)
 }
 
+# subclass of AbstractKeywordDocValuesField
+class org.elasticsearch.script.field.KeywordDocValuesField @dynamic_type {
+}
+
+# subclass of AbstractKeywordDocValuesField
+class org.elasticsearch.script.field.FlattenedDocValuesField @dynamic_type {
+}
+
 class org.elasticsearch.script.field.GeoPointDocValuesField @dynamic_type {
  GeoPoint get(GeoPoint)
  GeoPoint get(int, GeoPoint)

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml

@@ -41,6 +41,8 @@ setup:
  analyzer: standard
  rank:
  type: integer
+ flattended:
+ type: flattened
 
 
  - do:
@@ -1531,7 +1533,7 @@ setup:
 
  - do:
  index:
- index: test
+ index: versiontest
  id: 3000
  version: 50
  version_type: external
@@ -1543,6 +1545,7 @@ setup:
 
  - do:
  search:
+ index: versiontest
  rest_total_hits_as_int: true
  body:
  query: { term: { _id: 3000 } }
@@ -1554,11 +1557,11 @@ setup:
  script:
  source: "field('_seq_no').get(10000)"
  - match: { hits.hits.0.fields.ver.0: 50 }
- - match: { hits.hits.0.fields.seq.0: 3 }
+ - match: { hits.hits.0.fields.seq.0: 0 }
 
  - do:
  index:
- index: test
+ index: versiontest
  id: 3000
  version: 60
  version_type: external
@@ -1570,7 +1573,7 @@ setup:
  - do:
  catch: conflict
  index:
- index: test
+ index: versiontest
  id: 3000
  version: 55
  version_type: external
@@ -1581,6 +1584,7 @@ setup:
 
  - do:
  search:
+ index: versiontest
  rest_total_hits_as_int: true
  body:
  query: { term: { _id: 3000 } }
@@ -1592,4 +1596,103 @@ setup:
  script:
  source: "field('_seq_no').get(10000)"
  - match: { hits.hits.0.fields.ver.0: 60 }
- - match: { hits.hits.0.fields.seq.0: 4 }
+ - match: { hits.hits.0.fields.seq.0: 1 }
+
+---
+"flattened fields api":
+ - do:
+ indices.create:
+ index: flatindex
+ body:
+ settings:
+ number_of_shards: 1
+ mappings:
+ properties:
+ rank:
+ type: integer
+ flattened:
+ type: flattened
+
+ - do:
+ index:
+ index: flatindex
+ id: 40
+ body:
+ rank: 1
+ flattened:
+ top: 123
+ dict:
+ abc: def
+ hij: lmn
+ list: [true]
+
+ - do:
+ index:
+ index: flatindex
+ id: 41
+ body:
+ rank: 2
+ flattened:
+ top2: 876
+ dict2:
+ abc: def
+ opq: rst
+ hij: lmn
+ uvx: wyz
+ list2: [789, 1011, 1213]
+
+ - do:
+ indices.refresh: {}
+
+ - do:
+ search:
+ index: flatindex
+ rest_total_hits_as_int: true
+ body:
+ sort: [ { rank: asc } ]
+ script_fields:
+ f_root:
+ script:
+ source: "field('flattened').get('dne')"
+ f_root_index:
+ script:
+ source: "field('flattened').get(2, 'dne')"
+ f_top:
+ script:
+ source: "field('flattened.top').get('dne')"
+ f_top2:
+ script:
+ source: "field('flattened.top2').get('dne')"
+ f_dict:
+ script:
+ source: "field('flattened.dict.abc').get('dne')"
+ f_dict2:
+ script:
+ source: "field('flattened.dict2.uvx').get('dne')"
+ f_list:
+ script:
+ source: "field('flattened.list').get('dne')"
+ f_list2:
+ script:
+ source: "field('flattened.list2').get(2, 'dne')"
+ all:
+ script:
+ source: "String all = ''; for (String value : field('flattened')) { all += value } all"
+ - match: { hits.hits.0.fields.f_root.0: "123" }
+ - match: { hits.hits.0.fields.f_root_index.0: "lmn" }
+ - match: { hits.hits.0.fields.f_top.0: "123" }
+ - match: { hits.hits.0.fields.f_top2.0: "dne" }
+ - match: { hits.hits.0.fields.f_dict.0: "def" }
+ - match: { hits.hits.0.fields.f_dict2.0: "dne" }
+ - match: { hits.hits.0.fields.f_list.0: "true" }
+ - match: { hits.hits.0.fields.f_list2.0: "dne" }
+ - match: { hits.hits.0.fields.all.0: "123deflmntrue" }
+ - match: { hits.hits.1.fields.f_root.0: "1011" }
+ - match: { hits.hits.1.fields.f_root_index.0: "789" }
+ - match: { hits.hits.1.fields.f_top.0: "dne" }
+ - match: { hits.hits.1.fields.f_top2.0: "876" }
+ - match: { hits.hits.1.fields.f_dict.0: "dne" }
+ - match: { hits.hits.1.fields.f_dict2.0: "wyz" }
+ - match: { hits.hits.1.fields.f_list.0: "dne" }
+ - match: { hits.hits.1.fields.f_list2.0: "789" }
+ - match: { hits.hits.1.fields.all.0: "10111213789876deflmnrstwyz" }

server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java

@@ -41,7 +41,6 @@
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
 import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData;
 import org.elasticsearch.index.fielddata.LeafOrdinalsFieldData;
-import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
 import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
 import org.elasticsearch.index.mapper.DocumentParserContext;
@@ -58,7 +57,7 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
-import org.elasticsearch.script.field.DelegateDocValuesField;
+import org.elasticsearch.script.field.FlattenedDocValuesField;
 import org.elasticsearch.script.field.ToScriptField;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.MultiValueMode;
@@ -199,11 +198,7 @@ public FlattenedFieldMapper build(MapperBuilderContext context) {
  hasDocValues.get(),
  meta.get(),
  splitQueriesOnWhitespace.get(),
- eagerGlobalOrdinals.get(),
- (dv, n) -> new DelegateDocValuesField(
- new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))),
- n
- )
+ eagerGlobalOrdinals.get()
  );
  return new FlattenedFieldMapper(name, ft, this);
  }
@@ -367,14 +362,7 @@ public BytesRef indexedValueForSearch(Object value) {
  @Override
  public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
  failIfNoDocValues();
- return new KeyedFlattenedFieldData.Builder(
- name(),
- key,
- (dv, n) -> new DelegateDocValuesField(
- new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))),
- n
- )
- );
+ return new KeyedFlattenedFieldData.Builder(name(), key, (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n));
  }
 
  @Override
@@ -603,16 +591,14 @@ public IndexFieldData<?> build(IndexFieldDataCache cache, CircuitBreakerService
  public static final class RootFlattenedFieldType extends StringFieldType implements DynamicFieldType {
  private final boolean splitQueriesOnWhitespace;
  private final boolean eagerGlobalOrdinals;
- private final ToScriptField<SortedSetDocValues> toScriptField;
 
  public RootFlattenedFieldType(
  String name,
  boolean indexed,
  boolean hasDocValues,
  Map<String, String> meta,
  boolean splitQueriesOnWhitespace,
- boolean eagerGlobalOrdinals,
- ToScriptField<SortedSetDocValues> toScriptField
+ boolean eagerGlobalOrdinals
  ) {
  super(
  name,
@@ -624,7 +610,6 @@ public RootFlattenedFieldType(
  );
  this.splitQueriesOnWhitespace = splitQueriesOnWhitespace;
  this.eagerGlobalOrdinals = eagerGlobalOrdinals;
- this.toScriptField = toScriptField;
  }
 
  @Override
@@ -657,10 +642,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S
  return new SortedSetOrdinalsIndexFieldData.Builder(
  name(),
  CoreValuesSourceType.KEYWORD,
- (dv, n) -> new DelegateDocValuesField(
- new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))),
- n
- )
+ (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n)
  );
  }
 

server/src/main/java/org/elasticsearch/script/field/FlattenedDocValuesField.java

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.script.field;
+
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+
+public class FlattenedDocValuesField extends AbstractKeywordDocValuesField {
+ public FlattenedDocValuesField(SortedBinaryDocValues input, String name) {
+ super(input, name);
+ }
+}

server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java

@@ -8,7 +8,6 @@
 
 package org.elasticsearch.index.mapper.flattened;
 
-import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.DocValuesFieldExistsQuery;
 import org.apache.lucene.search.FuzzyQuery;
@@ -21,27 +20,19 @@
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.common.lucene.search.AutomatonQueries;
 import org.elasticsearch.common.unit.Fuzziness;
-import org.elasticsearch.index.fielddata.FieldData;
-import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
 import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType;
-import org.elasticsearch.script.field.DelegateDocValuesField;
-import org.elasticsearch.script.field.ToScriptField;
 
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 
 public class RootFlattenedFieldTypeTests extends FieldTypeTestCase {
- private static final ToScriptField<SortedSetDocValues> MOCK_TO_SCRIPT_FIELD = (dv, n) -> new DelegateDocValuesField(
- new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))),
- n
- );
 
  private static RootFlattenedFieldType createDefaultFieldType() {
- return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, MOCK_TO_SCRIPT_FIELD);
+ return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false);
  }
 
  public void testValueForDisplay() {
@@ -61,40 +52,16 @@ public void testTermQuery() {
  expected = AutomatonQueries.caseInsensitiveTermQuery(new Term("field", "Value"));
  assertEquals(expected, ft.termQueryCaseInsensitive("Value", null));
 
- RootFlattenedFieldType unsearchable = new RootFlattenedFieldType(
- "field",
- false,
- true,
- Collections.emptyMap(),
- false,
- false,
- MOCK_TO_SCRIPT_FIELD
- );
+ RootFlattenedFieldType unsearchable = new RootFlattenedFieldType("field", false, true, Collections.emptyMap(), false, false);
  IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null));
  assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
  }
 
  public void testExistsQuery() {
- RootFlattenedFieldType ft = new RootFlattenedFieldType(
- "field",
- true,
- false,
- Collections.emptyMap(),
- false,
- false,
- MOCK_TO_SCRIPT_FIELD
- );
+ RootFlattenedFieldType ft = new RootFlattenedFieldType("field", true, false, Collections.emptyMap(), false, false);
  assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))), ft.existsQuery(null));
 
- RootFlattenedFieldType withDv = new RootFlattenedFieldType(
- "field",
- true,
- true,
- Collections.emptyMap(),
- false,
- false,
- MOCK_TO_SCRIPT_FIELD
- );
+ RootFlattenedFieldType withDv = new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false);
  assertEquals(new DocValuesFieldExistsQuery("field"), withDv.existsQuery(null));
  }
 

x-pack/plugin/mapper-constant-keyword/src/main/resources/org/elasticsearch/xpack/constantkeyword/org.elasticsearch.xpack.constantkeyword.txt

@@ -6,7 +6,6 @@
 # Side Public License, v 1.
 #
 
+# subclass of AbstractKeywordDocValuesField
 class org.elasticsearch.xpack.constantkeyword.ConstantKeywordDocValuesField @dynamic_type {
- String get(String)
- String get(int, String)
 }