Merge pull request elastic#19920 from cbuescher/remove-SuggestUtil

Remove SuggestUtil helper class

Author: cbuescher

core/src/main/java/org/elasticsearch/search/suggest/DirectSpellcheckerSettings.java

@@ -21,8 +21,13 @@
 import org.apache.lucene.search.spell.DirectSpellChecker;
 import org.apache.lucene.search.spell.StringDistance;
 import org.apache.lucene.search.spell.SuggestMode;
+import org.apache.lucene.search.spell.SuggestWord;
+import org.apache.lucene.search.spell.SuggestWordFrequencyComparator;
+import org.apache.lucene.search.spell.SuggestWordQueue;
 import org.apache.lucene.util.automaton.LevenshteinAutomata;
 
+import java.util.Comparator;
+
 public class DirectSpellcheckerSettings {
 
  // NB: If this changes, make sure to change the default in TermBuilderSuggester
@@ -49,6 +54,9 @@ public class DirectSpellcheckerSettings {
  private int minWordLength = DEFAULT_MIN_WORD_LENGTH;
  private float minDocFreq = DEFAULT_MIN_DOC_FREQ;
 
+ private static final Comparator<SuggestWord> LUCENE_FREQUENCY = new SuggestWordFrequencyComparator();
+ private static final Comparator<SuggestWord> SCORE_COMPARATOR = SuggestWordQueue.DEFAULT_COMPARATOR;
+
  public SuggestMode suggestMode() {
  return suggestMode;
  }
@@ -129,6 +137,33 @@ public void minDocFreq(float minDocFreq) {
  this.minDocFreq = minDocFreq;
  }
 
+ public DirectSpellChecker createDirectSpellChecker() {
+
+ DirectSpellChecker directSpellChecker = new DirectSpellChecker();
+ directSpellChecker.setAccuracy(accuracy());
+ Comparator<SuggestWord> comparator;
+ switch (sort()) {
+ case SCORE:
+ comparator = SCORE_COMPARATOR;
+ break;
+ case FREQUENCY:
+ comparator = LUCENE_FREQUENCY;
+ break;
+ default:
+ throw new IllegalArgumentException("Illegal suggest sort: " + sort());
+ }
+ directSpellChecker.setComparator(comparator);
+ directSpellChecker.setDistance(stringDistance());
+ directSpellChecker.setMaxEdits(maxEdits());
+ directSpellChecker.setMaxInspections(maxInspections());
+ directSpellChecker.setMaxQueryFrequency(maxTermFreq());
+ directSpellChecker.setMinPrefix(prefixLength());
+ directSpellChecker.setMinQueryLength(minWordLength());
+ directSpellChecker.setThresholdFrequency(minDocFreq());
+ directSpellChecker.setLowerCaseTerms(false);
+ return directSpellChecker;
+ }
+
  @Override
  public String toString() {
  return "[" +

core/src/main/java/org/elasticsearch/search/suggest/SuggestUtils.java

@@ -37,7 +37,6 @@
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.index.query.QueryShardContext;
-import org.elasticsearch.search.suggest.SuggestUtils;
 import org.elasticsearch.search.suggest.SuggestionBuilder;
 import org.elasticsearch.search.suggest.SuggestionSearchContext.SuggestionContext;
 import org.elasticsearch.search.suggest.completion.context.ContextMapping;
@@ -48,7 +47,6 @@
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -90,10 +88,10 @@ public class CompletionSuggestionBuilder extends SuggestionBuilder<CompletionSug
  TLP_PARSER.declareField((parser, completionSuggestionContext, context) ->
  completionSuggestionContext.regexOptions = RegexOptions.parse(parser, context),
  RegexOptions.REGEX_OPTIONS, ObjectParser.ValueType.OBJECT);
- TLP_PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::field, SuggestUtils.Fields.FIELD);
- TLP_PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::analyzer, SuggestUtils.Fields.ANALYZER);
- TLP_PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::size, SuggestUtils.Fields.SIZE);
- TLP_PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::shardSize, SuggestUtils.Fields.SHARD_SIZE);
+ TLP_PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::field, FIELDNAME_FIELD);
+ TLP_PARSER.declareString(CompletionSuggestionBuilder.InnerBuilder::analyzer, ANALYZER_FIELD);
+ TLP_PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::size, SIZE_FIELD);
+ TLP_PARSER.declareInt(CompletionSuggestionBuilder.InnerBuilder::shardSize, SHARDSIZE_FIELD);
  TLP_PARSER.declareField((p, v, c) -> {
  // Copy the current structure. We will parse, once the mapping is provided
  XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
@@ -353,7 +351,7 @@ static CompletionSuggestionBuilder innerFromXContent(QueryParseContext parseCont
  // now we should have field name, check and copy fields over to the suggestion builder we return
  if (field == null) {
  throw new ElasticsearchParseException(
- "the required field option [" + SuggestUtils.Fields.FIELD.getPreferredName() + "] is missing");
+ "the required field option [" + FIELDNAME_FIELD.getPreferredName() + "] is missing");
  }
  return new CompletionSuggestionBuilder(field, builder);
  }

core/src/main/java/org/elasticsearch/search/suggest/completion/CompletionSuggestionBuilder.java

@@ -20,7 +20,6 @@
 
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
-import org.elasticsearch.search.suggest.SuggestUtils;
 import org.elasticsearch.search.suggest.phrase.DirectCandidateGenerator.Candidate;
 
 import java.util.Arrays;
@@ -73,7 +72,7 @@ public BytesRef join(BytesRef separator, BytesRefBuilder result, BytesRef preTag
  len += toJoin[i].length;
  }
  result.grow(len);
- return SuggestUtils.join(separator, result, toJoin);
+ return WordScorer.join(separator, result, toJoin);
  }
 
  /** Lower scores sorts first; if scores are equal,

core/src/main/java/org/elasticsearch/search/suggest/phrase/Correction.java

@@ -19,6 +19,10 @@
 package org.elasticsearch.search.suggest.phrase;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.Term;
@@ -29,8 +33,10 @@
 import org.apache.lucene.search.spell.SuggestWord;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.CharsRef;
 import org.apache.lucene.util.CharsRefBuilder;
-import org.elasticsearch.search.suggest.SuggestUtils;
+import org.apache.lucene.util.IOUtils;
+import org.elasticsearch.common.io.FastCharArrayReader;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -44,7 +50,7 @@
 import static java.lang.Math.max;
 import static java.lang.Math.round;
 
-final class DirectCandidateGenerator extends CandidateGenerator {
+public final class DirectCandidateGenerator extends CandidateGenerator {
 
  private final DirectSpellChecker spellchecker;
  private final String field;
@@ -140,7 +146,7 @@ protected BytesRef preFilter(final BytesRef term, final CharsRefBuilder spare, f
  return term;
  }
  final BytesRefBuilder result = byteSpare;
- SuggestUtils.analyze(preFilter, term, field, new SuggestUtils.TokenConsumer() {
+ analyze(preFilter, term, field, new TokenConsumer() {
 
  @Override
  public void nextToken() throws IOException {
@@ -156,7 +162,7 @@ protected void postFilter(final Candidate candidate, final CharsRefBuilder spare
  candidates.add(candidate);
  } else {
  final BytesRefBuilder result = byteSpare;
- SuggestUtils.analyze(postFilter, candidate.term, field, new SuggestUtils.TokenConsumer() {
+ analyze(postFilter, candidate.term, field, new TokenConsumer() {
  @Override
  public void nextToken() throws IOException {
  this.fillBytesRef(result);
@@ -189,6 +195,27 @@ protected long thresholdFrequency(long termFrequency, long dictionarySize) {
 
  }
 
+ public abstract static class TokenConsumer {
+ protected CharTermAttribute charTermAttr;
+ protected PositionIncrementAttribute posIncAttr;
+ protected OffsetAttribute offsetAttr;
+
+ public void reset(TokenStream stream) {
+ charTermAttr = stream.addAttribute(CharTermAttribute.class);
+ posIncAttr = stream.addAttribute(PositionIncrementAttribute.class);
+ offsetAttr = stream.addAttribute(OffsetAttribute.class);
+ }
+
+ protected BytesRef fillBytesRef(BytesRefBuilder spare) {
+ spare.copyChars(charTermAttr);
+ return spare.get();
+ }
+
+ public abstract void nextToken() throws IOException;
+
+ public void end() {}
+ }
+
  public static class CandidateSet {
  public Candidate[] candidates;
  public final Candidate originalTerm;
@@ -283,4 +310,40 @@ public Candidate createCandidate(BytesRef term, long frequency, double channelSc
  return new Candidate(term, frequency, channelScore, score(frequency, channelScore, dictSize), userInput);
  }
 
+ public static int analyze(Analyzer analyzer, BytesRef toAnalyze, String field, TokenConsumer consumer, CharsRefBuilder spare)
+ throws IOException {
+ spare.copyUTF8Bytes(toAnalyze);
+ CharsRef charsRef = spare.get();
+ try (TokenStream ts = analyzer.tokenStream(
+ field, new FastCharArrayReader(charsRef.chars, charsRef.offset, charsRef.length))) {
+ return analyze(ts, consumer);
+ }
+ }
+
+ /** NOTE: this method closes the TokenStream, even on exception, which is awkward
+ * because really the caller who called {@link Analyzer#tokenStream} should close it,
+ * but when trying that there are recursion issues when we try to use the same
+ * TokenStream twice in the same recursion... */
+ public static int analyze(TokenStream stream, TokenConsumer consumer) throws IOException {
+ int numTokens = 0;
+ boolean success = false;
+ try {
+ stream.reset();
+ consumer.reset(stream);
+ while (stream.incrementToken()) {
+ consumer.nextToken();
+ numTokens++;
+ }
+ consumer.end();
+ success = true;
+ } finally {
+ if (success) {
+ stream.close();
+ } else {
+ IOUtils.closeWhileHandlingException(stream);
+ }
+ }
+ return numTokens;
+ }
+
 }