Skip to content

Commit 9e2469e

Browse files
chrismalechrismale
committed
Add per-field Similarity support
1 parent 4e8a900 commit 9e2469e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+1137
-170
lines changed

‎src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1331,7 +1331,7 @@ private IndexWriter createWriter() throws IOException {
13311331
config.setIndexDeletionPolicy(deletionPolicy);
13321332
config.setMergeScheduler(mergeScheduler.newMergeScheduler());
13331333
config.setMergePolicy(mergePolicyProvider.newMergePolicy());
1334-
config.setSimilarity(similarityService.defaultIndexSimilarity());
1334+
config.setSimilarity(similarityService.similarity());
13351335
config.setRAMBufferSizeMB(indexingBufferSize.mbFrac());
13361336
config.setTermIndexInterval(termIndexInterval);
13371337
config.setReaderTermsIndexDivisor(termIndexDivisor);
@@ -1478,7 +1478,7 @@ class RobinSearchFactory extends SearcherFactory {
14781478
@Override
14791479
public IndexSearcher newSearcher(IndexReader reader) throws IOException {
14801480
IndexSearcher searcher = new IndexSearcher(reader);
1481-
searcher.setSimilarity(similarityService.defaultSearchSimilarity());
1481+
searcher.setSimilarity(similarityService.similarity());
14821482
if (warmer != null) {
14831483
// we need to pass a custom searcher that does not release anything on Engine.Search Release,
14841484
// we will release explicitly

‎src/main/java/org/elasticsearch/index/mapper/DocumentMapperParser.java‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import org.elasticsearch.index.mapper.object.ObjectMapper;
4646
import org.elasticsearch.index.mapper.object.RootObjectMapper;
4747
import org.elasticsearch.index.settings.IndexSettings;
48+
import org.elasticsearch.index.similarity.SimilarityLookupService;
4849

4950
import java.io.IOException;
5051
import java.util.Map;
@@ -58,6 +59,7 @@ public class DocumentMapperParser extends AbstractIndexComponent {
5859

5960
final AnalysisService analysisService;
6061
private final PostingsFormatService postingsFormatService;
62+
private final SimilarityLookupService similarityLookupService;
6163

6264
private final RootObjectMapper.TypeParser rootObjectTypeParser = new RootObjectMapper.TypeParser();
6365

@@ -66,14 +68,17 @@ public class DocumentMapperParser extends AbstractIndexComponent {
6668
private volatile ImmutableMap<String, Mapper.TypeParser> typeParsers;
6769
private volatile ImmutableMap<String, Mapper.TypeParser> rootTypeParsers;
6870

69-
public DocumentMapperParser(Index index, AnalysisService analysisService, PostingsFormatService postingsFormatService) {
70-
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS, analysisService, postingsFormatService);
71+
public DocumentMapperParser(Index index, AnalysisService analysisService, PostingsFormatService postingsFormatService,
72+
SimilarityLookupService similarityLookupService) {
73+
this(index, ImmutableSettings.Builder.EMPTY_SETTINGS, analysisService, postingsFormatService, similarityLookupService);
7174
}
7275

73-
public DocumentMapperParser(Index index, @IndexSettings Settings indexSettings, AnalysisService analysisService, PostingsFormatService postingsFormatService) {
76+
public DocumentMapperParser(Index index, @IndexSettings Settings indexSettings, AnalysisService analysisService,
77+
PostingsFormatService postingsFormatService, SimilarityLookupService similarityLookupService) {
7478
super(index, indexSettings);
7579
this.analysisService = analysisService;
7680
this.postingsFormatService = postingsFormatService;
81+
this.similarityLookupService = similarityLookupService;
7782
MapBuilder<String, Mapper.TypeParser> typeParsersBuilder = new MapBuilder<String, Mapper.TypeParser>()
7883
.put(ByteFieldMapper.CONTENT_TYPE, new ByteFieldMapper.TypeParser())
7984
.put(ShortFieldMapper.CONTENT_TYPE, new ShortFieldMapper.TypeParser())
@@ -132,7 +137,7 @@ public void putRootTypeParser(String type, Mapper.TypeParser typeParser) {
132137
}
133138

134139
public Mapper.TypeParser.ParserContext parserContext() {
135-
return new Mapper.TypeParser.ParserContext(postingsFormatService, analysisService, typeParsers);
140+
return new Mapper.TypeParser.ParserContext(postingsFormatService, analysisService, similarityLookupService, typeParsers);
136141
}
137142

138143
public DocumentMapper parse(String source) throws MapperParsingException {
@@ -166,7 +171,7 @@ public DocumentMapper parse(@Nullable String type, String source, String default
166171
}
167172
}
168173

169-
Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(postingsFormatService, analysisService, typeParsers);
174+
Mapper.TypeParser.ParserContext parserContext = new Mapper.TypeParser.ParserContext(postingsFormatService, analysisService, similarityLookupService, typeParsers);
170175

171176
DocumentMapper.Builder docBuilder = doc(index.name(), indexSettings, (RootObjectMapper.Builder) rootObjectTypeParser.parse(type, mapping, parserContext));
172177

‎src/main/java/org/elasticsearch/index/mapper/FieldMapper.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
3131
import org.elasticsearch.index.field.data.FieldDataType;
3232
import org.elasticsearch.index.query.QueryParseContext;
33+
import org.elasticsearch.index.similarity.SimilarityProvider;
3334

3435
/**
3536
*
@@ -157,6 +158,11 @@ public Term createIndexNameTerm(String value) {
157158
*/
158159
Analyzer searchQuoteAnalyzer();
159160

161+
/**
162+
* Similarity used for scoring queries on the field
163+
*/
164+
SimilarityProvider similarity();
165+
160166
/**
161167
* Returns the value that will be used as a result for search. Can be only of specific types... .
162168
*/

‎src/main/java/org/elasticsearch/index/mapper/Mapper.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.elasticsearch.common.xcontent.ToXContent;
2727
import org.elasticsearch.index.analysis.AnalysisService;
2828
import org.elasticsearch.index.codec.postingsformat.PostingsFormatService;
29+
import org.elasticsearch.index.similarity.SimilarityLookupService;
2930

3031
import java.io.IOException;
3132
import java.util.Map;
@@ -81,11 +82,15 @@ public static class ParserContext {
8182

8283
private final AnalysisService analysisService;
8384

85+
private final SimilarityLookupService similarityLookupService;
86+
8487
private final ImmutableMap<String, TypeParser> typeParsers;
8588

86-
public ParserContext(PostingsFormatService postingsFormatService, AnalysisService analysisService, ImmutableMap<String, TypeParser> typeParsers) {
89+
public ParserContext(PostingsFormatService postingsFormatService, AnalysisService analysisService,
90+
SimilarityLookupService similarityLookupService, ImmutableMap<String, TypeParser> typeParsers) {
8791
this.postingsFormatService = postingsFormatService;
8892
this.analysisService = analysisService;
93+
this.similarityLookupService = similarityLookupService;
8994
this.typeParsers = typeParsers;
9095
}
9196

@@ -97,6 +102,10 @@ public PostingsFormatService postingFormatService() {
97102
return postingsFormatService;
98103
}
99104

105+
public SimilarityLookupService similarityLookupService() {
106+
return similarityLookupService;
107+
}
108+
100109
public TypeParser typeParser(String type) {
101110
return typeParsers.get(Strings.toUnderscoreCase(type));
102111
}

‎src/main/java/org/elasticsearch/index/mapper/MapperService.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.elasticsearch.index.mapper.object.ObjectMapper;
5050
import org.elasticsearch.index.search.nested.NonNestedDocsFilter;
5151
import org.elasticsearch.index.settings.IndexSettings;
52+
import org.elasticsearch.index.similarity.SimilarityLookupService;
5253
import org.elasticsearch.indices.InvalidTypeNameException;
5354
import org.elasticsearch.indices.TypeMissingException;
5455

@@ -99,11 +100,12 @@ public class MapperService extends AbstractIndexComponent implements Iterable<Do
99100
private final SmartIndexNameSearchQuoteAnalyzer searchQuoteAnalyzer;
100101

101102
@Inject
102-
public MapperService(Index index, @IndexSettings Settings indexSettings, Environment environment, AnalysisService analysisService, PostingsFormatService postingsFormatService) {
103+
public MapperService(Index index, @IndexSettings Settings indexSettings, Environment environment, AnalysisService analysisService,
104+
PostingsFormatService postingsFormatService, SimilarityLookupService similarityLookupService) {
103105
super(index, indexSettings);
104106
this.analysisService = analysisService;
105107
this.postingsFormatService = postingsFormatService;
106-
this.documentParser = new DocumentMapperParser(index, indexSettings, analysisService, postingsFormatService);
108+
this.documentParser = new DocumentMapperParser(index, indexSettings, analysisService, postingsFormatService, similarityLookupService);
107109
this.searchAnalyzer = new SmartIndexNameSearchAnalyzer(analysisService.defaultSearchAnalyzer());
108110
this.searchQuoteAnalyzer = new SmartIndexNameSearchQuoteAnalyzer(analysisService.defaultSearchQuoteAnalyzer());
109111

‎src/main/java/org/elasticsearch/index/mapper/core/AbstractFieldMapper.java‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import org.elasticsearch.index.field.data.FieldDataType;
3838
import org.elasticsearch.index.mapper.*;
3939
import org.elasticsearch.index.query.QueryParseContext;
40+
import org.elasticsearch.index.similarity.SimilarityProvider;
4041

4142
import java.io.IOException;
4243

@@ -131,6 +132,11 @@ public T indexAnalyzer(NamedAnalyzer indexAnalyzer) {
131132
public T searchAnalyzer(NamedAnalyzer searchAnalyzer) {
132133
return super.searchAnalyzer(searchAnalyzer);
133134
}
135+
136+
@Override
137+
public T similarity(SimilarityProvider similarity) {
138+
return super.similarity(similarity);
139+
}
134140
}
135141

136142
public abstract static class Builder<T extends Builder, Y extends AbstractFieldMapper> extends Mapper.Builder<T, Y> {
@@ -144,6 +150,7 @@ public abstract static class Builder<T extends Builder, Y extends AbstractFieldM
144150
protected Boolean includeInAll;
145151
protected boolean indexOptionsSet = false;
146152
protected PostingsFormatProvider provider;
153+
protected SimilarityProvider similarity;
147154

148155
protected Builder(String name, FieldType fieldType) {
149156
super(name);
@@ -230,6 +237,11 @@ protected T postingsFormat(PostingsFormatProvider postingsFormat) {
230237
return builder;
231238
}
232239

240+
protected T similarity(SimilarityProvider similarity) {
241+
this.similarity = similarity;
242+
return builder;
243+
}
244+
233245
protected Names buildNames(BuilderContext context) {
234246
return new Names(name, buildIndexName(context), indexName == null ? name : indexName, buildFullName(context), context.path().sourcePath());
235247
}
@@ -250,9 +262,10 @@ protected String buildFullName(BuilderContext context) {
250262
protected final NamedAnalyzer indexAnalyzer;
251263
protected final NamedAnalyzer searchAnalyzer;
252264
protected PostingsFormatProvider postingsFormat;
265+
protected final SimilarityProvider similarity;
253266

254267
protected AbstractFieldMapper(Names names, float boost, FieldType fieldType, NamedAnalyzer indexAnalyzer,
255-
NamedAnalyzer searchAnalyzer, PostingsFormatProvider postingsFormat) {
268+
NamedAnalyzer searchAnalyzer, PostingsFormatProvider postingsFormat, SimilarityProvider similarity) {
256269
this.names = names;
257270
this.boost = boost;
258271
this.fieldType = fieldType;
@@ -276,6 +289,7 @@ protected AbstractFieldMapper(Names names, float boost, FieldType fieldType, Nam
276289
}
277290
}
278291
this.postingsFormat = postingsFormat;
292+
this.similarity = similarity;
279293
}
280294

281295
@Nullable
@@ -358,6 +372,11 @@ public Analyzer searchQuoteAnalyzer() {
358372
return this.searchAnalyzer;
359373
}
360374

375+
@Override
376+
public SimilarityProvider similarity() {
377+
return similarity;
378+
}
379+
361380
@Override
362381
public void parse(ParseContext context) throws IOException {
363382
try {
@@ -524,6 +543,17 @@ public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappi
524543
} else if (!this.searchAnalyzer.name().equals(fieldMergeWith.searchAnalyzer.name())) {
525544
mergeContext.addConflict("mapper [" + names.fullName() + "] has different search_analyzer");
526545
}
546+
547+
if (this.similarity == null) {
548+
if (fieldMergeWith.similarity() != null) {
549+
mergeContext.addConflict("mapper [" + names.fullName() + "] has different similarity");
550+
}
551+
} else if (fieldMergeWith.similarity() == null) {
552+
mergeContext.addConflict("mapper [" + names.fullName() + "] has different similarity");
553+
} else if (!this.similarity().equals(fieldMergeWith.similarity())) {
554+
mergeContext.addConflict("mapper [" + names.fullName() + "] has different similarity");
555+
}
556+
527557
if (!mergeContext.mergeFlags().simulate()) {
528558
// apply changeable values
529559
this.boost = fieldMergeWith.boost;

‎src/main/java/org/elasticsearch/index/mapper/core/BinaryFieldMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
120120
private long compressThreshold;
121121

122122
protected BinaryFieldMapper(Names names, FieldType fieldType, Boolean compress, long compressThreshold, PostingsFormatProvider provider) {
123-
super(names, 1.0f, fieldType, null, null, provider);
123+
super(names, 1.0f, fieldType, null, null, provider, null);
124124
this.compress = compress;
125125
this.compressThreshold = compressThreshold;
126126
}

‎src/main/java/org/elasticsearch/index/mapper/core/BooleanFieldMapper.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.elasticsearch.index.mapper.Mapper;
3333
import org.elasticsearch.index.mapper.MapperParsingException;
3434
import org.elasticsearch.index.mapper.ParseContext;
35+
import org.elasticsearch.index.similarity.SimilarityProvider;
3536

3637
import java.io.IOException;
3738
import java.util.Map;
@@ -113,10 +114,14 @@ public Builder indexName(String indexName) {
113114
return super.indexName(indexName);
114115
}
115116

117+
@Override
118+
public Builder similarity(SimilarityProvider similarity) {
119+
return super.similarity(similarity);
120+
}
116121

117122
@Override
118123
public BooleanFieldMapper build(BuilderContext context) {
119-
return new BooleanFieldMapper(buildNames(context), boost, fieldType, nullValue, provider);
124+
return new BooleanFieldMapper(buildNames(context), boost, fieldType, nullValue, provider, similarity);
120125
}
121126
}
122127

@@ -138,8 +143,8 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
138143

139144
private Boolean nullValue;
140145

141-
protected BooleanFieldMapper(Names names, float boost, FieldType fieldType, Boolean nullValue, PostingsFormatProvider provider) {
142-
super(names, boost, fieldType, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, provider);
146+
protected BooleanFieldMapper(Names names, float boost, FieldType fieldType, Boolean nullValue, PostingsFormatProvider provider, SimilarityProvider similarity) {
147+
super(names, boost, fieldType, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, provider, similarity);
143148
this.nullValue = nullValue;
144149
}
145150

@@ -238,6 +243,9 @@ protected void doXContentBody(XContentBuilder builder) throws IOException {
238243
if (indexOptions() != Defaults.BOOLEAN_FIELD_TYPE.indexOptions()) {
239244
builder.field("index_options", indexOptionToString(indexOptions()));
240245
}
246+
if (similarity() != null) {
247+
builder.field("similarity", similarity().name());
248+
}
241249
if (nullValue != null) {
242250
builder.field("null_value", nullValue);
243251
}

‎src/main/java/org/elasticsearch/index/mapper/core/ByteFieldMapper.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
import org.elasticsearch.index.mapper.*;
4343
import org.elasticsearch.index.query.QueryParseContext;
4444
import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
45+
import org.elasticsearch.index.similarity.SimilarityProvider;
4546

4647
import java.io.IOException;
4748
import java.util.Map;
@@ -86,7 +87,7 @@ public ByteFieldMapper build(BuilderContext context) {
8687
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
8788
ByteFieldMapper fieldMapper = new ByteFieldMapper(buildNames(context),
8889
precisionStep, fuzzyFactor, boost, fieldType, nullValue, ignoreMalformed(context),
89-
provider);
90+
provider, similarity);
9091
fieldMapper.includeInAll(includeInAll);
9192
return fieldMapper;
9293
}
@@ -113,10 +114,10 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
113114
private String nullValueAsString;
114115

115116
protected ByteFieldMapper(Names names, int precisionStep, String fuzzyFactor, float boost, FieldType fieldType,
116-
Byte nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider) {
117+
Byte nullValue, Explicit<Boolean> ignoreMalformed, PostingsFormatProvider provider, SimilarityProvider similarity) {
117118
super(names, precisionStep, fuzzyFactor, boost, fieldType,
118119
ignoreMalformed, new NamedAnalyzer("_byte/" + precisionStep, new NumericIntegerAnalyzer(precisionStep)),
119-
new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider);
120+
new NamedAnalyzer("_byte/max", new NumericIntegerAnalyzer(Integer.MAX_VALUE)), provider, similarity);
120121
this.nullValue = nullValue;
121122
this.nullValueAsString = nullValue == null ? null : nullValue.toString();
122123
}
@@ -354,6 +355,9 @@ protected void doXContentBody(XContentBuilder builder) throws IOException {
354355
if (fuzzyFactor != Defaults.FUZZY_FACTOR) {
355356
builder.field("fuzzy_factor", fuzzyFactor);
356357
}
358+
if (similarity() != null) {
359+
builder.field("similarity", similarity().name());
360+
}
357361
if (nullValue != null) {
358362
builder.field("null_value", nullValue);
359363
}

‎src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
import org.elasticsearch.index.mapper.*;
4646
import org.elasticsearch.index.query.QueryParseContext;
4747
import org.elasticsearch.index.search.NumericRangeFieldDataFilter;
48+
import org.elasticsearch.index.similarity.SimilarityProvider;
4849

4950
import java.io.IOException;
5051
import java.util.Map;
@@ -113,7 +114,7 @@ public DateFieldMapper build(BuilderContext context) {
113114
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
114115
DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter,
115116
precisionStep, fuzzyFactor, boost, fieldType, nullValue,
116-
timeUnit, parseUpperInclusive, ignoreMalformed(context), provider);
117+
timeUnit, parseUpperInclusive, ignoreMalformed(context), provider, similarity);
117118
fieldMapper.includeInAll(includeInAll);
118119
return fieldMapper;
119120
}
@@ -152,12 +153,12 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
152153
protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, String fuzzyFactor,
153154
float boost, FieldType fieldType,
154155
String nullValue, TimeUnit timeUnit, boolean parseUpperInclusive, Explicit<Boolean> ignoreMalformed,
155-
PostingsFormatProvider provider) {
156+
PostingsFormatProvider provider, SimilarityProvider similarity) {
156157
super(names, precisionStep, fuzzyFactor, boost, fieldType,
157158
ignoreMalformed, new NamedAnalyzer("_date/" + precisionStep,
158159
new NumericDateAnalyzer(precisionStep, dateTimeFormatter.parser())),
159160
new NamedAnalyzer("_date/max", new NumericDateAnalyzer(Integer.MAX_VALUE, dateTimeFormatter.parser())),
160-
provider);
161+
provider, similarity);
161162
this.dateTimeFormatter = dateTimeFormatter;
162163
this.nullValue = nullValue;
163164
this.timeUnit = timeUnit;
@@ -428,6 +429,9 @@ protected void doXContentBody(XContentBuilder builder) throws IOException {
428429
builder.field("fuzzy_factor", fuzzyFactor);
429430
}
430431
builder.field("format", dateTimeFormatter.format());
432+
if (similarity() != null) {
433+
builder.field("similarity", similarity().name());
434+
}
431435
if (nullValue != null) {
432436
builder.field("null_value", nullValue);
433437
}

0 commit comments

Comments
 (0)