elastic#2436 expose KeepWordTokenFilter by default

Author: s1monw

src/main/java/org/elasticsearch/index/analysis/AnalysisModule.java

@@ -480,6 +480,7 @@ public void processTokenFilters(TokenFiltersBindings tokenFiltersBindings) {
  tokenFiltersBindings.processTokenFilter("word_delimiter", WordDelimiterTokenFilterFactory.class);
  tokenFiltersBindings.processTokenFilter("synonym", SynonymTokenFilterFactory.class);
  tokenFiltersBindings.processTokenFilter("elision", ElisionTokenFilterFactory.class);
+ tokenFiltersBindings.processTokenFilter("keep", KeepWordFilterFactory.class);
 
  tokenFiltersBindings.processTokenFilter("pattern_replace", PatternReplaceTokenFilterFactory.class);
  tokenFiltersBindings.processTokenFilter("dictionary_decompounder", DictionaryCompoundWordTokenFilterFactory.class);

src/main/java/org/elasticsearch/index/analysis/KeepWordFilterFactory.java

@@ -0,0 +1,97 @@
+package org.elasticsearch.index.analysis;
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import java.util.Arrays;
+import java.util.Map;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.miscellaneous.KeepWordFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.inject.assistedinject.Assisted;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.env.Environment;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.settings.IndexSettings;
+import org.elasticsearch.indices.analysis.IndicesAnalysisService;
+
+/**
+ * A {@link TokenFilterFactory} for {@link KeepWordFilter}. This filter only
+ * keep tokens that are contained in the term set configured via
+ * {@value #KEEP_WORDS_KEY} setting. This filter acts like an inverse stop
+ * filter.
+ * 
+ * Configuration options:
+ * 
+ * <ul>
+ * <li>{@value #KEEP_WORDS_KEY} the array of words / tokens to keep.</li>
+ * 
+ * <li>{@value #KEEP_WORDS_PATH_KEY} an reference to a file containing the words
+ * / tokens to keep. Note: this is an alternative to {@value #KEEP_WORDS_KEY} if
+ * both are set an exception will be thrown.</li>
+ * 
+ * <li>{@value #ENABLE_POS_INC_KEY} <code>true</code> iff the filter should
+ * maintain position increments for dropped tokens. The default is
+ * <code>true</code>.</li>
+ * 
+ * <li>{@value #KEEP_WORDS_CASE_KEY} to use case sensitive keep words. The
+ * default is <code>false</code> which corresponds to case-sensitive.</li>
+ * </ul>
+ * 
+ * @see StopTokenFilterFactory
+ * 
+ */
+@AnalysisSettingsRequired
+public class KeepWordFilterFactory extends AbstractTokenFilterFactory {
+ private Boolean enablePositionIncrements;
+ private CharArraySet keepWords;
+ private static final String KEEP_WORDS_KEY = "keep_words";
+ private static final String KEEP_WORDS_PATH_KEY = KEEP_WORDS_KEY + "_path";
+ private static final String KEEP_WORDS_CASE_KEY = KEEP_WORDS_KEY + "_case"; // for javadoc
+ private static final String ENABLE_POS_INC_KEY = "enable_position_increments";
+ 
+ @Inject
+ public KeepWordFilterFactory(Index index, @IndexSettings Settings indexSettings, 
+ Environment env, IndicesAnalysisService indicesAnalysisService, Map<String, TokenizerFactoryFactory> tokenizerFactories,
+ @Assisted String name, @Assisted Settings settings) {
+ super(index, indexSettings, name, settings);
+ 
+ final String[] arrayKeepWords = settings.getAsArray(KEEP_WORDS_KEY);
+ final String keepWordsPath = settings.get(KEEP_WORDS_PATH_KEY, null);
+ if (!(arrayKeepWords == null ^ keepWordsPath == null)) {
+ // we don't allow both or non
+ throw new ElasticSearchIllegalArgumentException("keep requires either `" + KEEP_WORDS_KEY + "` or `"
+ + KEEP_WORDS_PATH_KEY + "` to be configured");
+ }
+ this.enablePositionIncrements = settings.getAsBoolean(ENABLE_POS_INC_KEY, true);
+ this.keepWords = Analysis.getWordSet(env, settings, KEEP_WORDS_KEY, version);
+ 
+ }
+ 
+ @Override
+ public TokenStream create(TokenStream tokenStream) {
+ return new KeepWordFilter(enablePositionIncrements, tokenStream, keepWords);
+ }
+
+ 
+ 
+ 
+
+}

src/test/java/org/elasticsearch/test/unit/index/analysis/AnalysisTestsHelper.java

@@ -21,6 +21,7 @@
 
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.elasticsearch.common.inject.Injector;
 import org.elasticsearch.common.inject.ModulesBuilder;
 import org.elasticsearch.common.settings.ImmutableSettings;
@@ -45,6 +46,11 @@ public static AnalysisService createAnalysisServiceFromClassPath(String resource
  Settings settings = ImmutableSettings.settingsBuilder()
  .loadFromClasspath(resource).build();
 
+ return createAnalysisServiceFromSettings(settings);
+ }
+
+ public static AnalysisService createAnalysisServiceFromSettings(
+ Settings settings) {
  Index index = new Index("test");
 
  Injector parentInjector = new ModulesBuilder().add(new SettingsModule(settings),
@@ -71,4 +77,19 @@ public static void assertSimpleTSOutput(TokenStream stream, String[] expected) t
  }
  Assert.assertEquals(i, expected.length, "not all tokens produced");
  }
+ 
+ public static void assertSimpleTSOutput(TokenStream stream, String[] expected, int[] posInc) throws IOException {
+ stream.reset();
+ CharTermAttribute termAttr = stream.getAttribute(CharTermAttribute.class);
+ PositionIncrementAttribute posIncAttr = stream.getAttribute(PositionIncrementAttribute.class);
+ Assert.assertNotNull(termAttr);
+ int i = 0;
+ while (stream.incrementToken()) {
+ Assert.assertTrue(i < expected.length, "got extra term: " + termAttr.toString());
+ Assert.assertEquals(termAttr.toString(), expected[i], "expected different term at index " + i);
+ Assert.assertEquals(posIncAttr.getPositionIncrement(), posInc[i]);
+ i++;
+ }
+ Assert.assertEquals(i, expected.length, "not all tokens produced");
+ }
 }

src/test/java/org/elasticsearch/test/unit/index/analysis/KeepFilterFactoryTests.java

@@ -0,0 +1,90 @@
+/*
+ * Licensed to ElasticSearch and Shay Banon under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. ElasticSearch licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.test.unit.index.analysis;
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.util.Version;
+import org.elasticsearch.ElasticSearchIllegalArgumentException;
+import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.analysis.AnalysisService;
+import org.elasticsearch.index.analysis.KeepWordFilterFactory;
+import org.elasticsearch.index.analysis.ShingleTokenFilterFactory;
+import org.elasticsearch.index.analysis.TokenFilterFactory;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+
+public class KeepFilterFactoryTests {
+
+ private static final String RESOURCE = "org/elasticsearch/test/unit/index/analysis/keep_analysis.json";
+ 
+ 
+ @Test
+ public void testLoadWithoutSettings() {
+ AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(RESOURCE);
+ TokenFilterFactory tokenFilter = analysisService.tokenFilter("keep");
+ Assert.assertNull(tokenFilter);
+ }
+ 
+ @Test
+ public void testLoadOverConfiguredSettings() {
+ Settings settings = ImmutableSettings.settingsBuilder()
+ .put("index.analysis.filter.broken_keep_filter.type", "keep")
+ .put("index.analysis.filter.broken_keep_filter.keep_words_path", "does/not/exists.txt")
+ .put("index.analysis.filter.broken_keep_filter.keep_words", "[\"Hello\", \"worlD\"]")
+ .build();
+ try {
+ AnalysisTestsHelper.createAnalysisServiceFromSettings(settings);
+ Assert.fail("path and array are configured");
+ } catch (Exception e) {
+ assertThat(e.getCause(), instanceOf(ElasticSearchIllegalArgumentException.class));
+ }
+ }
+
+ @Test
+ public void testCaseInsensitiveMapping() throws IOException {
+ AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(RESOURCE);
+ TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_keep_filter");
+ assertThat(tokenFilter, instanceOf(KeepWordFilterFactory.class));
+ String source = "hello small world";
+ String[] expected = new String[]{"hello", "world"};
+ Tokenizer tokenizer = new WhitespaceTokenizer(Version.LUCENE_40, new StringReader(source));
+ AnalysisTestsHelper.assertSimpleTSOutput(tokenFilter.create(tokenizer), expected, new int[] {1,2});
+ }
+
+ @Test
+ public void testCaseSensitiveMapping() throws IOException {
+ AnalysisService analysisService = AnalysisTestsHelper.createAnalysisServiceFromClassPath(RESOURCE);
+ TokenFilterFactory tokenFilter = analysisService.tokenFilter("my_case_sensitive_keep_filter");
+ assertThat(tokenFilter, instanceOf(KeepWordFilterFactory.class));
+ String source = "Hello small world";
+ String[] expected = new String[]{"Hello"};
+ Tokenizer tokenizer = new WhitespaceTokenizer(Version.LUCENE_40, new StringReader(source));
+ AnalysisTestsHelper.assertSimpleTSOutput(tokenFilter.create(tokenizer), expected, new int[] {1});
+ }
+
+}

src/test/java/org/elasticsearch/test/unit/index/analysis/keep_analysis.json

@@ -0,0 +1,19 @@
+{
+ "index":{
+ "analysis":{
+ "filter":{
+ "my_keep_filter":{
+ "type":"keep",
+ "keep_words" : ["Hello", "worlD"],
+ "enable_position_increments" : true,
+ "keep_words_case" : true
+ },
+ "my_case_sensitive_keep_filter":{
+ "type":"keep",
+ "keep_words" : ["Hello", "worlD"],
+ "enable_position_increments" : false
+ }
+ }
+ }
+ }
+}