Fixed unable to highlight on all multi-valued field values.

Closes elastic#2384

Author: martijnvg

src/main/java/org/apache/lucene/search/vectorhighlight/AbstractFragmentsBuilder.java

@@ -35,20 +35,20 @@ public abstract class AbstractFragmentsBuilder extends BaseFragmentsBuilder {
 
  private boolean discreteMultiValueHighlighting = true;
 
- protected AbstractFragmentsBuilder(){
+ protected AbstractFragmentsBuilder() {
  super();
  }
 
- protected AbstractFragmentsBuilder(BoundaryScanner boundaryScanner){
+ protected AbstractFragmentsBuilder(BoundaryScanner boundaryScanner) {
  super(boundaryScanner);
  }
 
- protected AbstractFragmentsBuilder( String[] preTags, String[] postTags ){
+ protected AbstractFragmentsBuilder(String[] preTags, String[] postTags) {
  super(preTags, postTags);
  }
 
  public AbstractFragmentsBuilder(String[] preTags, String[] postTags, BoundaryScanner bs) {
- super( preTags, postTags, bs );
+ super(preTags, postTags, bs);
  }
 
  public void setDiscreteMultiValueHighlighting(boolean discreteMultiValueHighlighting) {
@@ -62,7 +62,6 @@ public String[] createFragments(IndexReader reader, int docId,
  throw new IllegalArgumentException("maxNumFragments(" + maxNumFragments + ") must be positive number.");
  }
 
- List<String> fragments = new ArrayList<String>(maxNumFragments);
  List<FieldFragList.WeightedFragInfo> fragInfos = fieldFragList.getFragInfos();
  Field[] values = getFields(reader, docId, fieldName);
  if (values.length == 0) {
@@ -75,9 +74,12 @@ public String[] createFragments(IndexReader reader, int docId,
 
  fragInfos = getWeightedFragInfoList(fragInfos);
 
+ int limitFragments = maxNumFragments < fragInfos.size() ? maxNumFragments : fragInfos.size();
+ List<String> fragments = new ArrayList<String>(limitFragments);
+
  StringBuilder buffer = new StringBuilder();
  int[] nextValueIndex = {0};
- for (int n = 0; n < maxNumFragments && n < fragInfos.size(); n++) {
+ for (int n = 0; n < limitFragments; n++) {
  FieldFragList.WeightedFragInfo fragInfo = fragInfos.get(n);
  fragments.add(makeFragment(buffer, nextValueIndex, values, fragInfo, preTags, postTags, encoder));
  }

src/main/java/org/elasticsearch/search/highlight/HighlightPhase.java

@@ -304,9 +304,10 @@ public int compare(TextFragment o1, TextFragment o2) {
  String[] fragments;
 
  // a HACK to make highlighter do highlighting, even though its using the single frag list builder
- int numberOfFragments = field.numberOfFragments() == 0 ? 1 : field.numberOfFragments();
+ int numberOfFragments = field.numberOfFragments() == 0 ? Integer.MAX_VALUE : field.numberOfFragments();
+ int fragmentCharSize = field.numberOfFragments() == 0 ? Integer.MAX_VALUE : field.fragmentCharSize();
  // we highlight against the low level reader and docId, because if we load source, we want to reuse it if possible
- fragments = cache.fvh.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(), mapper.names().indexName(), field.fragmentCharSize(), numberOfFragments,
+ fragments = cache.fvh.getBestFragments(fieldQuery, hitContext.reader(), hitContext.docId(), mapper.names().indexName(), fragmentCharSize, numberOfFragments,
  entry.fragListBuilder, entry.fragmentsBuilder, field.preTags(), field.postTags(), encoder);
 
  if (fragments != null && fragments.length > 0) {

src/test/java/org/elasticsearch/test/integration/search/highlight/HighlighterSearchTests.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.indices.IndexMissingException;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
@@ -887,4 +888,35 @@ public void testDisableFastVectorHighlighter() throws Exception {
  assertThat(hit.highlightFields().get("title").fragments()[0].string(), equalTo("This is a <em>test</em> for the <em>workaround</em> for the fast vector highlighting SOLR-3724"));
  }
  }
+
+ @Test
+ public void testFSHHighlightAllMvFragments() throws Exception {
+ try {
+ client.admin().indices().prepareDelete("test").execute().actionGet();
+ } catch (Exception e) {
+ // ignore
+ }
+
+ client.admin().indices().prepareCreate("test").setSettings(ImmutableSettings.settingsBuilder()
+ .put("number_of_shards", 1).put("number_of_replicas", 0))
+ .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("properties")
+ .startObject("tags").field("type", "string").field("term_vector", "with_positions_offsets").endObject()
+ .endObject().endObject().endObject())
+ .execute().actionGet();
+
+ client.prepareIndex("test", "type1", "1")
+ .setSource(jsonBuilder().startObject().field("tags",
+ "this is a really long tag i would like to highlight",
+ "here is another one that is very long and has the tag token near the end").endObject())
+ .setRefresh(true).execute().actionGet();
+
+ SearchResponse response = client.prepareSearch("test")
+ .setQuery(QueryBuilders.matchQuery("tags", "tag"))
+ .addHighlightedField("tags", -1, 0)
+ .execute().actionGet();
+
+ assertThat(2, equalTo(response.hits().hits()[0].highlightFields().get("tags").fragments().length));
+ assertThat("this is a really long <em>tag</em> i would like to highlight", equalTo(response.hits().hits()[0].highlightFields().get("tags").fragments()[0].string()));
+ assertThat("here is another one that is very long and has the <em>tag</em> token near the end", equalTo(response.hits().hits()[0].highlightFields().get("tags").fragments()[1].string()));
+ }
 }