Add support for maps with enum keys and overflow properties maps

Author: swallez

java-client/src/main/java/co/elastic/clients/base/rest_client/RestClientTransport.java

@@ -43,6 +43,7 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.Function;
@@ -168,8 +169,8 @@ private <RequestT> org.elasticsearch.client.Request prepareLowLevelRequest(
  // Request has a body and must implement JsonpSerializable or NdJsonpSerializable
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
 
- if (request instanceof NdJsonpSerializable<?>) {
- writeNdJson((NdJsonpSerializable<?>) request, baos);
+ if (request instanceof NdJsonpSerializable) {
+ writeNdJson((NdJsonpSerializable) request, baos);
  } else {
  JsonGenerator generator = mapper.jsonProvider().createGenerator(baos);
  mapper.serialize(request, generator);
@@ -184,15 +185,15 @@ private <RequestT> org.elasticsearch.client.Request prepareLowLevelRequest(
  }
 
  /**
- * Write an nd-json value by serializing each of its items on a separate line.
- * <p>
- * If an item itself implements {@link NdJsonpSerializable}, it is output as nd-json. This allows flattening
- * nested structures.
+ * Write an nd-json value by serializing each of its items on a separate line, recursing if its items themselves implement
+ * {@link NdJsonpSerializable} to flattening nested structures.
  */
- private void writeNdJson(NdJsonpSerializable<?> value, ByteArrayOutputStream baos) {
- for (Object item: value) {
- if (item instanceof NdJsonpSerializable<?>) {
- writeNdJson((NdJsonpSerializable<?>) item, baos);
+ private void writeNdJson(NdJsonpSerializable value, ByteArrayOutputStream baos) {
+ Iterator<?> values = value._serializables();
+ while(values.hasNext()) {
+ Object item = values.next();
+ if (item instanceof NdJsonpSerializable && item != value) { // do not recurse on the item itself
+ writeNdJson((NdJsonpSerializable) item, baos);
  } else {
  JsonGenerator generator = mapper.jsonProvider().createGenerator(baos);
  mapper.serialize(item, generator);

java-client/src/main/java/co/elastic/clients/json/DelegatingDeserializer.java

@@ -41,11 +41,12 @@ public abstract <FieldType> void add(
  * This structure is flattened in the corresponding Java classes, and this method should be used to register
  * its setter.
  *
- * @param keySetter the key setter
+ * @param setter the key setter
+ * @param deserializer the key deserializer (from a KEY_NAME event)
  */
- public abstract void setKey(
- BiConsumer<ObjectType, String> keySetter
- );
+ public <FieldType> void setKey(BiConsumer<ObjectType, FieldType> setter, JsonpDeserializer<FieldType> deserializer) {
+ throw new UnsupportedOperationException();
+ }
 
  /**
  * Used for internally tagged variants containers to indicate the object's property that defines the variant type

java-client/src/main/java/co/elastic/clients/json/JsonpDeserializer.java

@@ -20,6 +20,7 @@
 package co.elastic.clients.json;
 
 import co.elastic.clients.util.ObjectBuilder;
+import co.elastic.clients.util.StringEnum;
 import co.elastic.clients.util.TriFunction;
 import jakarta.json.JsonNumber;
 import jakarta.json.JsonValue;
@@ -161,7 +162,8 @@ public T deserialize(JsonParser parser, JsonpMapper mapper, Event event) {
  //----- Lazily created parser
 
  public static <T> JsonpDeserializer<T> lazy(Supplier<JsonpDeserializer<T>> builder) {
- return new LazyDeserializer<>(builder, EnumSet.of(Event.START_OBJECT));
+ //return new LazyDeserializer<>(builder, EnumSet.of(Event.START_OBJECT));
+ return new LazyDeserializer<>(builder, EnumSet.allOf(Event.class));
  }
 
  public static <T> JsonpDeserializer<T> lazy(Supplier<JsonpDeserializer<T>> builder, EnumSet<Event> acceptedEvents) {
@@ -278,7 +280,9 @@ public static JsonpDeserializer<String> stringDeserializer() {
 
  private static final JsonpDeserializer<String> STRING =
  // String parsing is lenient and accepts any other primitive type
- new JsonpDeserializer<String>(EnumSet.of(Event.VALUE_STRING, Event.VALUE_NUMBER, Event.VALUE_FALSE, Event.VALUE_TRUE)) {
+ new JsonpDeserializer<String>(
+ EnumSet.of(Event.KEY_NAME, Event.VALUE_STRING, Event.VALUE_NUMBER, Event.VALUE_FALSE, Event.VALUE_TRUE)
+ ) {
  @Override
  public String deserialize(JsonParser parser, JsonpMapper mapper, Event event) {
  if (event == Event.VALUE_TRUE) {
@@ -468,4 +472,33 @@ public Map<String, T> deserialize(JsonParser parser, JsonpMapper mapper, Event e
  public static <T> JsonpDeserializer<Map<String, T>> stringMapDeserializer(JsonpDeserializer<T> itemDeserializer) {
  return new StringMapDeserializer<T>(itemDeserializer);
  }
+
+ private static class EnumMapDeserializer<K, V> extends JsonpDeserializer<Map<K, V>> {
+ private final JsonpDeserializer<K> keyDeserializer;
+ private final JsonpDeserializer<V> valueDeserializer;
+
+ protected EnumMapDeserializer(JsonpDeserializer<K> keyDeserializer, JsonpDeserializer<V> valueDeserializer) {
+ super(EnumSet.of(Event.START_OBJECT));
+ this.keyDeserializer = keyDeserializer;
+ this.valueDeserializer = valueDeserializer;
+ }
+
+ @Override
+ public Map<K, V> deserialize(JsonParser parser, JsonpMapper mapper, Event event) {
+ Map<K, V> result = new HashMap<>();
+ while ((event = parser.next()) != Event.END_OBJECT) {
+ JsonpUtils.expectEvent(parser, Event.KEY_NAME, event);
+ K key = keyDeserializer.deserialize(parser, mapper, event);
+ V value = valueDeserializer.deserialize(parser, mapper);
+ result.put(key, value);
+ }
+ return result;
+ }
+ }
+
+ public static <K extends StringEnum, V> JsonpDeserializer<Map<K, V>> enumMapDeserializer(
+ JsonpDeserializer<K> keyDeserializer, JsonpDeserializer<V> valueDeserializer
+ ) {
+ return new EnumMapDeserializer<K, V>(keyDeserializer, valueDeserializer);
+ }
 }

java-client/src/main/java/co/elastic/clients/json/NdJsonpSerializable.java

@@ -19,8 +19,15 @@
 
 package co.elastic.clients.json;
 
+import java.util.Iterator;
+
 /**
  * Marks a class a being serialized as nd-json (e.g. bulk requests).
+ * <p>
+ * If an item returned by the iterator implements {@link NdJsonpSerializable}, it should also be output as nd-json.
+ * This allows flattening nested structures (e.g. a bulk index operation and its document). However, if this object returns itself
+ * as part of the iterator values, it then has to be serialized and not flattened again (which would lead to an infinite loop).
  */
-public interface NdJsonpSerializable<T> extends Iterable<T> {
+public interface NdJsonpSerializable {
+ Iterator<?> _serializables();
 }

java-client/src/main/java/co/elastic/clients/json/ObjectDeserializer.java

@@ -44,7 +44,10 @@ public FieldDeserializer(String name, String[] deprecatedNames) {
  this.name = name;
  this.deprecatedNames = deprecatedNames;
  }
+
  public abstract void deserialize(JsonParser parser, JsonpMapper mapper, String fieldName, ObjectType object);
+
+ public abstract void deserialize(JsonParser parser, JsonpMapper mapper, String fieldName, ObjectType object, Event event);
  }
 
  /** Field deserializer for objects (and boxed primitives) */
@@ -69,26 +72,39 @@ public void deserialize(JsonParser parser, JsonpMapper mapper, String fieldName,
  FieldType fieldValue = deserializer.deserialize(parser, mapper);
  setter.accept(object, fieldValue);
  }
+
+ public void deserialize(JsonParser parser, JsonpMapper mapper, String fieldName, ObjectType object, Event event) {
+ deserializer.ensureAccepts(parser, event);
+ FieldType fieldValue = deserializer.deserialize(parser, mapper, event);
+ setter.accept(object, fieldValue);
+ }
  }
 
  private static final FieldDeserializer<?> IGNORED_FIELD = new FieldDeserializer<Object>("-", null) {
+
  @Override
  public void deserialize(JsonParser parser, JsonpMapper mapper, String fieldName, Object object) {
  JsonpUtils.skipValue(parser);
  }
+
+ @Override
+ public void deserialize(JsonParser parser, JsonpMapper mapper, String fieldName, Object object, Event event) {
+ JsonpUtils.skipValue(parser, event);
+ }
  };
 
  //---------------------------------------------------------------------------------------------
 
  private final Supplier<ObjectType> constructor;
  protected final Map<String, FieldDeserializer<ObjectType>> fieldDeserializers;
- private BiConsumer<ObjectType, String> keySetter;
+ private FieldDeserializer<ObjectType> singleKey;
  private String typeProperty;
  private FieldDeserializer<ObjectType> shortcutProperty;
  private QuadConsumer<ObjectType, String, JsonParser, JsonpMapper> unknownFieldHandler;
 
  public ObjectDeserializer(Supplier<ObjectType> constructor) {
- super(EnumSet.of(Event.START_OBJECT));
+ //super(EnumSet.of(Event.START_OBJECT));
+ super(EnumSet.allOf(Event.class));
  this.constructor = constructor;
  this.fieldDeserializers = new HashMap<>();
  }
@@ -98,51 +114,54 @@ public ObjectType deserialize(JsonParser parser, JsonpMapper mapper, Event event
  }
 
  public ObjectType deserialize(ObjectType value, JsonParser parser, JsonpMapper mapper, Event event) {
- ensureAccepts(parser, event);
  if (event == Event.VALUE_NULL) {
  return null;
  }
 
- if (event != Event.START_OBJECT && shortcutProperty != null) {
- shortcutProperty.deserialize(parser, mapper, shortcutProperty.name, value);
+ if (singleKey != null) {
+ // There's a wrapping property whose name is the key value
+ event = JsonpUtils.expectNextEvent(parser, Event.KEY_NAME);
+ singleKey.deserialize(parser, mapper, null, value, event);
+ event = parser.next();
  }
 
- if (keySetter != null) {
- String key = JsonpUtils.expectKeyName(parser, parser.next());
- keySetter.accept(value, key);
- JsonpUtils.expectNextEvent(parser, Event.START_OBJECT);
- }
+ if (shortcutProperty != null && event != Event.START_OBJECT) {
+ // This is the shortcut property (should be a value event, this will be checked by its deserializer)
+ shortcutProperty.deserialize(parser, mapper, shortcutProperty.name, value, event);
 
- if (typeProperty == null) {
- // Regular object: read all properties until we reach the end of the object
- while ((event = parser.next()) != Event.END_OBJECT) {
+ } else {
+ JsonpUtils.expectEvent(parser, Event.START_OBJECT, event);
+ if (typeProperty == null) {
+ // Regular object: read all properties until we reach the end of the object
+ while ((event = parser.next()) != Event.END_OBJECT) {
+
+ JsonpUtils.expectEvent(parser, Event.KEY_NAME, event);
+ String fieldName = parser.getString();
+
+ FieldDeserializer<ObjectType> fieldDeserializer = fieldDeserializers.get(fieldName);
+ if (fieldDeserializer == null) {
+ parseUnknownField(parser, mapper, fieldName, value);
+ } else {
+ fieldDeserializer.deserialize(parser, mapper, fieldName, value);
+ }
+ }
 
- JsonpUtils.expectEvent(parser, Event.KEY_NAME, event);
- String fieldName = parser.getString();
+ } else {
+ // Union variant: find the property to find the proper deserializer
+ Map.Entry<String, JsonParser> unionInfo = JsonpUtils.lookAheadFieldValue(typeProperty, parser, mapper);
+ String variant = unionInfo.getKey();
+ JsonParser innerParser = unionInfo.getValue();
 
- FieldDeserializer<ObjectType> fieldDeserializer = fieldDeserializers.get(fieldName);
+ FieldDeserializer<ObjectType> fieldDeserializer = fieldDeserializers.get(variant);
  if (fieldDeserializer == null) {
- parseUnknownField(parser, mapper, fieldName, value);
+ parseUnknownField(parser, mapper, variant, value);
  } else {
- fieldDeserializer.deserialize(parser, mapper, fieldName, value);
+ fieldDeserializer.deserialize(innerParser, mapper, variant, value);
  }
  }
-
- } else {
- // Union variant: find the property to find the proper deserializer
- Map.Entry<String, JsonParser> unionInfo = JsonpUtils.lookAheadFieldValue(typeProperty, parser, mapper);
- String variant = unionInfo.getKey();
- JsonParser innerParser = unionInfo.getValue();
-
- FieldDeserializer<ObjectType> fieldDeserializer = fieldDeserializers.get(variant);
- if (fieldDeserializer == null) {
- parseUnknownField(parser, mapper, variant, value);
- } else {
- fieldDeserializer.deserialize(innerParser, mapper, variant, value);
- }
  }
 
- if (keySetter != null) {
+ if (singleKey != null) {
  JsonpUtils.expectNextEvent(parser, Event.END_OBJECT);
  }
 
@@ -198,8 +217,8 @@ public <FieldType> void add(
  }
 
  @Override
- public void setKey(BiConsumer<ObjectType, String> setter) {
- this.keySetter = setter;
+ public <FieldType> void setKey(BiConsumer<ObjectType, FieldType> setter, JsonpDeserializer<FieldType> deserializer) {
+ this.singleKey = new FieldObjectDeserializer<>(setter, deserializer, null, null);
  }
 
  @Override

java-client/src/main/java/co/elastic/clients/json/ValueBodyDeserializer.java

@@ -71,15 +71,6 @@ public <FieldType> void add(
  this.valueDeserializer = valueParser;
  }
 
- /**
- * Not supported in this implementation
- * @throws UnsupportedOperationException not implemented.
- */
- @Override
- public void setKey(BiConsumer<ObjectType, String> setter) {
- throw new UnsupportedOperationException();
- }
-
  /**
  * Not supported in this implementation
  * @throws UnsupportedOperationException not implemented.

java-client/src/main/java/co/elastic/clients/util/StringEnum.java

@@ -46,7 +46,7 @@ class Deserializer<T extends Enum<T> & StringEnum> extends JsonpDeserializer<T>
  private final Map<String, T> lookupTable;
 
  public Deserializer(T[] values) {
- super(EnumSet.of(JsonParser.Event.VALUE_STRING));
+ super(EnumSet.of(JsonParser.Event.VALUE_STRING, JsonParser.Event.KEY_NAME));
 
  // Use the same size calculation as in java.lang.Enum.enumConstantDirectory
  this.lookupTable = new HashMap<>((int)(values.length / 0.75f) + 1);

java-client/src/main/java/co/elastic/clients/util/TaggedUnionUtils.java

@@ -19,6 +19,11 @@
 
 package co.elastic.clients.util;
 
+import co.elastic.clients.json.NdJsonpSerializable;
+
+import java.util.Collections;
+import java.util.Iterator;
+
 public class TaggedUnionUtils {
  public static <V, U extends TaggedUnion<?>> V get(U union, String type) {
  if (union._is(type)) {
@@ -29,4 +34,35 @@ public static <V, U extends TaggedUnion<?>> V get(U union, String type) {
  throw new IllegalStateException("Cannot get '" + type + "' variant: current variant is '" + union._type() + "'.");
  }
  }
+
+ public static <T> Iterator<?> ndJsonIterator(TaggedUnion<T> union) {
+
+ T value = union._get();
+
+ if (value instanceof NdJsonpSerializable) {
+ // Iterate on value's items, replacing value, if it appears, by the union. This allows JSON wrapping
+ // done by the container to happen.
+ Iterator<?> valueIterator = ((NdJsonpSerializable) value)._serializables();
+
+ return new Iterator<Object>() {
+ @Override
+ public boolean hasNext() {
+ return valueIterator.hasNext();
+ }
+
+ @Override
+ public Object next() {
+ Object next = valueIterator.next();
+ if (next == value) {
+ return union;
+ } else {
+ return next;
+ }
+ }
+ };
+ } else {
+ // Nothing to flatten
+ return Collections.singletonList(union).iterator();
+ }
+ }
 }

java-client/src/test/java/co/elastic/clients/elasticsearch/end_to_end/RequestTest.java

@@ -30,7 +30,7 @@
 import co.elastic.clients.elasticsearch.core.GetResponse;
 import co.elastic.clients.elasticsearch.core.IndexResponse;
 import co.elastic.clients.elasticsearch.core.SearchResponse;
-import co.elastic.clients.elasticsearch.core.bulk.ResponseItem;
+import co.elastic.clients.elasticsearch.core.bulk.OperationType;
 import co.elastic.clients.elasticsearch.cat.NodesResponse;
 import co.elastic.clients.elasticsearch.indices.CreateIndexResponse;
 import co.elastic.clients.elasticsearch.indices.GetIndexResponse;
@@ -208,20 +208,20 @@ public void testBulkRequest() throws IOException {
  appData.setMsg("Some message");
 
  BulkResponse bulk = client.bulk(_0 -> _0
- .addOperation(_1 -> _1
+ .addOperations(_1 -> _1
  .create(_2 -> _2
  .index("foo")
  .id("abc")
+ .document(appData)
  )
  )
- .addDocument(appData)
  );
 
  assertFalse(bulk.errors());
  assertEquals(1, bulk.items().size());
- assertEquals(ResponseItem.CREATE, bulk.items().get(0)._type());
- assertEquals("foo", bulk.items().get(0).create().index());
- assertEquals(1L, bulk.items().get(0).create().version().longValue());
+ assertEquals(OperationType.Create, bulk.items().get(0).operationType());
+ assertEquals("foo", bulk.items().get(0).index());
+ assertEquals(1L, bulk.items().get(0).version().longValue());
  }
 
  @Test