reuse non analyzed token stream for string types

so heavyweight token stream won't be created each time

Author: kimchy

src/main/java/org/elasticsearch/index/mapper/core/StringFieldMapper.java

@@ -20,6 +20,9 @@
 package org.elasticsearch.index.mapper.core;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -294,7 +297,7 @@ protected Field parseCreateField(ParseContext context) throws IOException {
  context.ignoredValue(names.indexName(), value);
  return null;
  }
- Field field = new Field(names.indexName(), value, fieldType);
+ Field field = new StringField(names.indexName(), value, fieldType);
  field.setBoost(boost);
  return field;
  }
@@ -364,4 +367,87 @@ protected void doXContentBody(XContentBuilder builder) throws IOException {
  builder.field("ignore_above", ignoreAbove);
  }
  }
+
+ /**
+ * Extension of {@link Field} supporting reuse of a cached TokenStream for not-tokenized values.
+ */
+ static class StringField extends Field {
+
+ public StringField(String name, String value, FieldType fieldType) {
+ super(name, value, fieldType);
+ }
+
+ @Override
+ public TokenStream tokenStream(Analyzer analyzer) throws IOException {
+ if (!fieldType().indexed()) {
+ return null;
+ }
+ // Only use the cached TokenStream if the value is indexed and not-tokenized
+ if (fieldType().tokenized()) {
+ return super.tokenStream(analyzer);
+ }
+ return NOT_ANALYZED_TOKENSTREAM.get().setValue((String) fieldsData);
+ }
+ }
+
+ private static final ThreadLocal<StringTokenStream> NOT_ANALYZED_TOKENSTREAM = new ThreadLocal<StringTokenStream>() {
+ @Override
+ protected StringTokenStream initialValue() {
+ return new StringTokenStream();
+ }
+ };
+
+
+ // Copied from Field.java
+ static final class StringTokenStream extends TokenStream {
+ private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
+ private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
+ private boolean used = false;
+ private String value = null;
+
+ /**
+ * Creates a new TokenStream that returns a String as single token.
+ * <p>Warning: Does not initialize the value, you must call
+ * {@link #setValue(String)} afterwards!
+ */
+ StringTokenStream() {
+ }
+
+ /**
+ * Sets the string value.
+ */
+ StringTokenStream setValue(String value) {
+ this.value = value;
+ return this;
+ }
+
+ @Override
+ public boolean incrementToken() {
+ if (used) {
+ return false;
+ }
+ clearAttributes();
+ termAttribute.append(value);
+ offsetAttribute.setOffset(0, value.length());
+ used = true;
+ return true;
+ }
+
+ @Override
+ public void end() {
+ final int finalOffset = value.length();
+ offsetAttribute.setOffset(finalOffset, finalOffset);
+ value = null;
+ }
+
+ @Override
+ public void reset() {
+ used = false;
+ }
+
+ @Override
+ public void close() {
+ value = null;
+ }
+ }
 }