fix properly handling acceptDocs in filters

our idea is to apply it on the "filtered/constant" level, and not on compound filters, so we won't apply it multiple times. The solution is conservative a bit now, we can further optimize it in the future, for example, not to wrap it when no caching is done within the filter chain

Author: kimchy

src/main/java/org/apache/lucene/queryparser/classic/ExistsFieldQueryExtension.java

@@ -22,7 +22,7 @@
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermRangeFilter;
-import org.elasticsearch.common.lucene.search.DeletionAwareConstantScoreQuery;
+import org.elasticsearch.common.lucene.search.XConstantScoreQuery;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryParseContext;
 
@@ -54,6 +54,6 @@ public Query query(QueryParseContext parseContext, String queryText) {
 
  filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext);
 
- return new DeletionAwareConstantScoreQuery(filter);
+ return new XConstantScoreQuery(filter);
  }
 }

src/main/java/org/apache/lucene/queryparser/classic/MissingFieldQueryExtension.java

@@ -22,8 +22,8 @@
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermRangeFilter;
-import org.elasticsearch.common.lucene.search.DeletionAwareConstantScoreQuery;
 import org.elasticsearch.common.lucene.search.NotFilter;
+import org.elasticsearch.common.lucene.search.XConstantScoreQuery;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.query.QueryParseContext;
 
@@ -59,6 +59,6 @@ public Query query(QueryParseContext parseContext, String queryText) {
 
  filter = wrapSmartNameFilter(filter, smartNameFieldMappers, parseContext);
 
- return new DeletionAwareConstantScoreQuery(filter);
+ return new XConstantScoreQuery(filter);
  }
 }

src/main/java/org/apache/lucene/search/vectorhighlight/CustomFieldQuery.java

@@ -24,10 +24,7 @@
 import org.apache.lucene.queries.FilterClause;
 import org.apache.lucene.search.*;
 import org.apache.lucene.search.spans.SpanTermQuery;
-import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
-import org.elasticsearch.common.lucene.search.TermFilter;
-import org.elasticsearch.common.lucene.search.XBooleanFilter;
-import org.elasticsearch.common.lucene.search.XTermsFilter;
+import org.elasticsearch.common.lucene.search.*;
 import org.elasticsearch.common.lucene.search.function.FiltersFunctionScoreQuery;
 import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
 
@@ -87,6 +84,9 @@ void flatten(Query sourceQuery, IndexReader reader, Collection<Query> flatQuerie
  } else if (sourceQuery instanceof FilteredQuery) {
  flatten(((FilteredQuery) sourceQuery).getQuery(), reader, flatQueries);
  flatten(((FilteredQuery) sourceQuery).getFilter(), reader, flatQueries);
+ } else if (sourceQuery instanceof XFilteredQuery) {
+ flatten(((XFilteredQuery) sourceQuery).getQuery(), reader, flatQueries);
+ flatten(((XFilteredQuery) sourceQuery).getFilter(), reader, flatQueries);
  } else if (sourceQuery instanceof MultiPhrasePrefixQuery) {
  try {
  flatten(sourceQuery.rewrite(reader), reader, flatQueries);

src/main/java/org/elasticsearch/common/lucene/search/NotDeletedFilter.java renamed to src/main/java/org/elasticsearch/common/lucene/search/ApplyAcceptedDocsFilter.java

@@ -20,24 +20,23 @@
 package org.elasticsearch.common.lucene.search;
 
 import org.apache.lucene.index.AtomicReaderContext;
-import org.apache.lucene.search.DocIdSet;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.search.Filter;
-import org.apache.lucene.search.FilteredDocIdSetIterator;
+import org.apache.lucene.search.*;
 import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.FixedBitSet;
 import org.elasticsearch.common.lucene.docset.DocIdSets;
 
 import java.io.IOException;
 
 /**
- * A filter that filters out deleted documents.
+ * The assumption is that the underlying filter might not apply the accepted docs, so this filter helps to wrap
+ * the actual filter and apply the actual accepted docs.
  */
-public class NotDeletedFilter extends Filter {
+// TODO: we can try and be smart, and only apply if if a filter is cached (down the "chain") since that's the only place that acceptDocs are not applied in ES
+public class ApplyAcceptedDocsFilter extends Filter {
 
  private final Filter filter;
 
- public NotDeletedFilter(Filter filter) {
+ public ApplyAcceptedDocsFilter(Filter filter) {
  this.filter = filter;
  }
 
@@ -47,10 +46,14 @@ public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws
  if (DocIdSets.isEmpty(docIdSet)) {
  return null;
  }
- if (!context.reader().hasDeletions()) {
+ if (acceptDocs == null) {
  return docIdSet;
  }
- return new NotDeletedDocIdSet(docIdSet, context.reader().getLiveDocs());
+ if (acceptDocs == context.reader().getLiveDocs()) {
+ // optimized wrapper for not deleted cases
+ return new NotDeletedDocIdSet(docIdSet, acceptDocs);
+ }
+ return BitsFilteredDocIdSet.wrap(docIdSet, acceptDocs);
  }
 
  public Filter filter() {
@@ -59,7 +62,7 @@ public Filter filter() {
 
  @Override
  public String toString() {
- return "NotDeleted(" + filter + ")";
+ return filter.toString();
  }
 
  static class NotDeletedDocIdSet extends DocIdSet {

src/main/java/org/elasticsearch/common/lucene/search/Queries.java

@@ -32,7 +32,7 @@
 public class Queries {
 
  // We don't use MatchAllDocsQuery, its slower than the one below ... (much slower)
- public final static Query MATCH_ALL_QUERY = new DeletionAwareConstantScoreQuery(new MatchAllDocsFilter());
+ public final static Query MATCH_ALL_QUERY = new XConstantScoreQuery(new MatchAllDocsFilter());
  public final static Query NO_MATCH_QUERY = MatchNoDocsQuery.INSTANCE;
 
  /**
@@ -113,8 +113,8 @@ public static boolean isConstantMatchAllQuery(Query query) {
  if (query == Queries.MATCH_ALL_QUERY) {
  return true;
  }
- if (query instanceof DeletionAwareConstantScoreQuery) {
- DeletionAwareConstantScoreQuery scoreQuery = (DeletionAwareConstantScoreQuery) query;
+ if (query instanceof XConstantScoreQuery) {
+ XConstantScoreQuery scoreQuery = (XConstantScoreQuery) query;
  if (scoreQuery.getFilter() instanceof MatchAllDocsFilter) {
  return true;
  }

src/main/java/org/elasticsearch/common/lucene/search/DeletionAwareConstantScoreQuery.java renamed to src/main/java/org/elasticsearch/common/lucene/search/XConstantScoreQuery.java

@@ -25,12 +25,12 @@
 /**
  * We still need sometimes to exclude deletes, because we don't remove them always with acceptDocs on filters
  */
-public class DeletionAwareConstantScoreQuery extends ConstantScoreQuery {
+public class XConstantScoreQuery extends ConstantScoreQuery {
 
  private final Filter actualFilter;
 
- public DeletionAwareConstantScoreQuery(Filter filter) {
- super(new NotDeletedFilter(filter));
+ public XConstantScoreQuery(Filter filter) {
+ super(new ApplyAcceptedDocsFilter(filter));
  this.actualFilter = filter;
  }