Add support for a new output format : GeoJSON

README.md

@@ -4,7 +4,7 @@ Elasticsearch Data Format Plugin
 ## Overview
 
 Elasticsearch Data Format Plugin provides a feature to allow you to download a response of a search result as several formats other than JSON.
-The supported formats are CSV, Excel, JSON(Bulk) and JSON(Object List).
+The supported formats are CSV, Excel, JSON(Bulk), JSON(Object List) and GeoJSON.
 
 ## Version
 
@@ -31,7 +31,7 @@ If not, it's as scan query(all data are stored.).
 | Request Parameter | Type | Description |
 |:------------------|:-------:|:------------|
 | append.header | boolean | Append column headers if true |
-| fields_name | string | choose the fields to dump |
+| fields_name | string | choose the fields to dump (comma separate format) |
 | source | string | [Query DSL](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html) |
 | csv.separator | string | Separate character in CSV |
 | csv.quote | string | Quote character in CSV|
@@ -46,7 +46,7 @@ If not, it's as scan query(all data are stored.).
 | Request Parameter | Type | Description |
 |:------------------|:-------:|:------------|
 | append.header | boolean | Append column headers if true |
-| fields_name | string | choose the fields to dump |
+| fields_name | string | choose the fields to dump (comma separate format) |
 | source | string | [Query DSL](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html) |
 
 ### Excel 2007
@@ -55,6 +55,8 @@ If not, it's as scan query(all data are stored.).
 
 | Request Parameter | Type | Description |
 |:------------------|:-------:|:------------|
+| append.header | boolean | Append column headers if true |
+| fields_name | string | choose the fields to dump (comma separate format) |
 | source | string | [Query DSL](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html) |
 
 ### JSON (Elasticsearch Bulk format)
@@ -75,3 +77,19 @@ If not, it's as scan query(all data are stored.).
 | :---------------- | :----: | :----------------------------------------------------------- |
 | source | string | [Query DSL](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html) |
 
+### GeoJSON (Open GIS standard)
+
+ $ curl -o /tmp/data.json -XGET "localhost:9200/{index}/{type}/_data?format=geojson&source=..."
+
+| Request Parameter | Type | Description |
+| :----------------------- | :----: | :----------------------------------------------------------- |
+| source | string | [Query DSL](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html) |
+| geometry.lon_field | string | Longitude field for coordinates (Support Geometry type "Point") |
+| geometry.lat_field | string | Latitude field for coordinates (Support Geometry type "Point") |
+| geometry.alt_field | string | Altitude field for coordinates (Support Geometry type "Point") |
+| geometry.coord_field | string | Coordinates field. Support all Geometry types (see [GeoJSON Example](https://en.wikipedia.org/wiki/GeoJSON)).<br/>If set, overwrite `geometry.lon_field`, `geometry.lat_field` and `geometry.alt_field` |
+| geometry.type_field | string | Geometry type field (see [GeoJSON Example](https://en.wikipedia.org/wiki/GeoJSON))<br/>Only used if `geometry.coord_field` param is set |
+| keep_geometry_info | boolean | Keep or not the original geometry fields in final GeoJSON properties (default: false) |
+| exclude_fields | string | Exclude fields in final geojson properties (comma separate format) |
+
+**NB**: Field name can use basic style like `a` or JSONpath style like `a.b.c[2].d`

pom.xml

@@ -121,6 +121,11 @@
 <artifactId>poi-ooxml-schemas</artifactId>
 <version>${poi.version}</version>
 </dependency>
+<dependency>
+<groupId>com.google.code.gson</groupId>
+<artifactId>gson</artifactId>
+<version>2.8.6</version>
+</dependency>
 <dependency>
 <groupId>org.codelibs</groupId>
 <artifactId>elasticsearch-cluster-runner</artifactId>

src/main/java/org/codelibs/elasticsearch/df/content/ContentType.java

@@ -1,6 +1,7 @@
 package org.codelibs.elasticsearch.df.content;
 
 import org.codelibs.elasticsearch.df.content.csv.CsvContent;
+import org.codelibs.elasticsearch.df.content.geojson.GeoJsonContent;
 import org.codelibs.elasticsearch.df.content.json.JsonContent;
 import org.codelibs.elasticsearch.df.content.json.JsonListContent;
 import org.codelibs.elasticsearch.df.content.xls.XlsContent;
@@ -117,6 +118,27 @@ public String fileName(final RestRequest request) {
  }
  return index + ".json";
  }
+ },
+ GEOJSON(60) {
+ @Override
+ public String contentType() {
+ return "application/geo+json";
+ }
+
+ @Override
+ public DataContent dataContent(final Client client,
+ final RestRequest request) {
+ return new GeoJsonContent(client, request, this);
+ }
+
+ @Override
+ public String fileName(final RestRequest request) {
+ final String index = request.param("index");
+ if (index == null) {
+ return "_all.geojson";
+ }
+ return index + ".geojson";
+ }
  };
 
  private int index;

src/main/java/org/codelibs/elasticsearch/df/content/geojson/GeoJsonContent.java

@@ -0,0 +1,240 @@
+package org.codelibs.elasticsearch.df.content.geojson;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.codelibs.elasticsearch.df.content.ContentType;
+import org.codelibs.elasticsearch.df.content.DataContent;
+import org.codelibs.elasticsearch.df.util.JsonUtils;
+import org.codelibs.elasticsearch.df.util.RequestUtil;
+import org.codelibs.elasticsearch.df.util.StringUtils;
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.xcontent.XContentHelper;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.rest.RestChannel;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.SearchHits;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+
+public class GeoJsonContent extends DataContent {
+ private static final Logger logger = LogManager.getLogger(GeoJsonContent.class);
+
+ private final String geometryCoordinatesLonField;
+ private final String geometryCoordinatesLatField;
+ private final String geometryCoordinatesAltField;
+ private final String geometryTypeField;
+ private final String geometryCoordinatesField;
+ private final boolean geometryKeepGeoInfo;
+ private final List<String> excludeFields;
+
+ public GeoJsonContent(final Client client, final RestRequest request, final ContentType contentType) {
+ super(client, request, contentType);
+
+ geometryCoordinatesLonField = request.param("geometry.lon_field",StringUtils.EMPTY_STRING);
+ geometryCoordinatesLatField = request.param("geometry.lat_field",StringUtils.EMPTY_STRING);
+ geometryCoordinatesAltField = request.param("geometry.alt_field",StringUtils.EMPTY_STRING);
+ geometryTypeField = request.param("geometry.type_field",StringUtils.EMPTY_STRING);
+ geometryCoordinatesField = request.param("geometry.coord_field",StringUtils.EMPTY_STRING);
+ geometryKeepGeoInfo = request.paramAsBoolean("keep_geometry_info",false);
+
+ final String[] fields = request.paramAsStringArray("exclude_fields", StringUtils.EMPTY_STRINGS);
+ if (fields.length == 0) {
+ excludeFields = new ArrayList<>();
+ } else {
+ final List<String> fieldList = new ArrayList<>();
+ for (final String field : fields) {
+ fieldList.add(field.trim());
+ }
+ excludeFields = Collections.unmodifiableList(fieldList);
+ }
+
+ if (logger.isDebugEnabled()) {
+ logger.debug("geometryTypeField: {}, geometryCoordinatesField: {}, geometryCoordinatesLonField: {}, " +
+ "geometryCoordinatesLatField: {}, geometryCoordinatesAltField: {}, geometryKeepGeoInfo: {}, excludeFields: {}",
+ geometryTypeField, geometryCoordinatesField, geometryCoordinatesLonField, 
+ geometryCoordinatesLatField, geometryCoordinatesAltField, geometryKeepGeoInfo, excludeFields);
+ }
+ }
+
+ @Override
+ public void write(final File outputFile, final SearchResponse response, final RestChannel channel,
+ final ActionListener<Void> listener) {
+ try {
+ final OnLoadListener onLoadListener = new OnLoadListener(
+ outputFile, listener);
+ onLoadListener.onResponse(response);
+ } catch (final Exception e) {
+ listener.onFailure(new ElasticsearchException("Failed to write data.",
+ e));
+ }
+ }
+
+ protected class OnLoadListener implements ActionListener<SearchResponse> {
+ protected ActionListener<Void> listener;
+
+ protected Writer writer;
+
+ protected File outputFile;
+
+ private long currentCount = 0;
+
+ private boolean firstLine = true;
+
+ protected OnLoadListener(final File outputFile, final ActionListener<Void> listener) {
+ this.outputFile = outputFile;
+ this.listener = listener;
+ try {
+ writer = new BufferedWriter(new OutputStreamWriter(
+ new FileOutputStream(outputFile), "UTF-8"));
+ } catch (final Exception e) {
+ throw new ElasticsearchException("Could not open "
+ + outputFile.getAbsolutePath(), e);
+ }
+ try {
+ writer.append("{\"type\": \"FeatureCollection\", \"features\": [");
+ }catch (final Exception e) {
+ onFailure(e);
+ }
+ }
+
+ @Override
+ public void onResponse(final SearchResponse response) {
+ final Gson gsonWriter = new GsonBuilder().create();
+ final String scrollId = response.getScrollId();
+ final SearchHits hits = response.getHits();
+ final int size = hits.getHits().length;
+ currentCount += size;
+ if (logger.isDebugEnabled()) {
+ logger.debug("scrollId: {}, totalHits: {}, hits: {}, current: {}",
+ scrollId, hits.getTotalHits(), size, currentCount);
+ }
+ try {
+ for (final SearchHit hit : hits) {
+ final String source = XContentHelper.convertToJson(
+ hit.getSourceRef(), true, false, XContentType.JSON);
+ if (!firstLine){
+ writer.append(',');
+ }else{
+ firstLine = false;
+ }
+
+ final JsonElement propertiesJson = JsonParser.parseString(source);
+ String geometryType = "";
+
+ JsonArray geometryCoordinates = new JsonArray();
+ if (!geometryCoordinatesField.isEmpty()){
+ JsonElement jsonEltCoord = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesField);
+ if (jsonEltCoord !=null && !jsonEltCoord.isJsonNull()){
+ geometryCoordinates = jsonEltCoord.getAsJsonArray​();
+ if (!geometryKeepGeoInfo){
+ JsonUtils.removeJsonElement(propertiesJson,geometryCoordinatesField);
+ }
+ }
+ if (!geometryTypeField.isEmpty()){
+ JsonElement jsonEltType = JsonUtils.getJsonElement(propertiesJson,geometryTypeField);
+ if (jsonEltType !=null && !jsonEltType.isJsonNull()){
+ geometryType = jsonEltType.getAsString();
+ if (!geometryKeepGeoInfo){
+ JsonUtils.removeJsonElement(propertiesJson,geometryTypeField);
+ }
+ }
+ }
+ }else{
+ if (!geometryCoordinatesLonField.isEmpty() && !geometryCoordinatesLatField.isEmpty()){
+ JsonElement jsonEltLon = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesLonField);
+ JsonElement jsonEltLat = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesLatField);
+ if (jsonEltLon !=null && !jsonEltLon.isJsonNull() && jsonEltLat !=null && !jsonEltLat.isJsonNull()){
+ geometryCoordinates.add(jsonEltLon.getAsNumber());
+ geometryCoordinates.add(jsonEltLat.getAsNumber());
+ if (!geometryKeepGeoInfo) {
+ JsonUtils.removeJsonElement(propertiesJson,geometryCoordinatesLonField);
+ JsonUtils.removeJsonElement(propertiesJson,geometryCoordinatesLatField);
+ }
+ }
+ }
+ if (!geometryCoordinatesAltField.isEmpty()){
+ JsonElement jsonElt = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesAltField);
+ if (jsonElt !=null && !jsonElt.isJsonNull()){
+ geometryCoordinates.add(jsonElt.getAsNumber());
+ if (!geometryKeepGeoInfo) {
+ JsonUtils.removeJsonElement(propertiesJson,geometryCoordinatesAltField);
+ }
+ }
+ }
+ geometryType = "Point";
+ }
+
+ for (String excludeField : excludeFields) {
+ JsonUtils.removeJsonElement(propertiesJson,excludeField);
+ }
+
+ JsonObject geometryObject = new JsonObject();
+ geometryObject.addProperty("type", geometryType);
+ geometryObject.add("coordinates", geometryCoordinates);
+
+ JsonObject featureObject = new JsonObject();
+ featureObject.addProperty("type", "Feature");
+ featureObject.add("geometry", geometryObject);
+ featureObject.add("properties", propertiesJson.getAsJsonObject());
+
+ writer.append('\n').append(gsonWriter.toJson(featureObject));
+ }
+
+ if (size == 0 || scrollId == null) {
+ // end
+ writer.append('\n').append("]}");
+ writer.flush();
+ close();
+ listener.onResponse(null);
+ } else {
+ client.prepareSearchScroll(scrollId)
+ .setScroll(RequestUtil.getScroll(request))
+ .execute(this);
+ }
+ } catch (final Exception e) {
+ onFailure(e);
+ }
+ }
+
+ @Override
+ public void onFailure(final Exception e) {
+ try {
+ close();
+ } catch (final Exception e1) {
+ // ignore
+ }
+ listener.onFailure(new ElasticsearchException("Failed to write data.",
+ e));
+ }
+
+ private void close() {
+ if (writer != null) {
+ try {
+ writer.close();
+ } catch (final IOException e) {
+ throw new ElasticsearchException("Could not close "
+ + outputFile.getAbsolutePath(), e);
+ }
+ }
+ }
+ }
+}

src/main/java/org/codelibs/elasticsearch/df/rest/RestDataAction.java

@@ -132,6 +132,10 @@ private ContentType getContentType(final RestRequest request) {
  } else if ("application/list+json".equals(contentType)
  || "jsonlist".equals(contentType)) {
  return ContentType.JSONLIST;
+ } else if ("application/geo+json".equals(contentType)
+ || "application/geojson".equals(contentType)
+ || "geojson".equals(contentType)) {
+ return ContentType.GEOJSON;
  }
 
  return null;