LOOKUP JOIN using field-caps for field mapping

docs/changelog/117246.yaml

@@ -0,0 +1,5 @@
+pr: 117246
+summary: LOOKUP JOIN using field-caps for field mapping
+area: ES|QL
+type: enhancement
+issues: []

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java

@@ -56,6 +56,8 @@ public class CsvTestsDataLoader {
  private static final TestsDataset APPS = new TestsDataset("apps");
  private static final TestsDataset APPS_SHORT = APPS.withIndex("apps_short").withTypeMapping(Map.of("id", "short"));
  private static final TestsDataset LANGUAGES = new TestsDataset("languages");
+ private static final TestsDataset LANGUAGES_LOOKUP = LANGUAGES.withIndex("languages_lookup")
+ .withSetting("languages_lookup-settings.json");
  private static final TestsDataset ALERTS = new TestsDataset("alerts");
  private static final TestsDataset UL_LOGS = new TestsDataset("ul_logs");
  private static final TestsDataset SAMPLE_DATA = new TestsDataset("sample_data");
@@ -93,14 +95,13 @@ public class CsvTestsDataLoader {
  private static final TestsDataset BOOKS = new TestsDataset("books");
  private static final TestsDataset SEMANTIC_TEXT = new TestsDataset("semantic_text").withInferenceEndpoint(true);
 
- private static final String LOOKUP_INDEX_SUFFIX = "_lookup";
-
  public static final Map<String, TestsDataset> CSV_DATASET_MAP = Map.ofEntries(
  Map.entry(EMPLOYEES.indexName, EMPLOYEES),
  Map.entry(HOSTS.indexName, HOSTS),
  Map.entry(APPS.indexName, APPS),
  Map.entry(APPS_SHORT.indexName, APPS_SHORT),
  Map.entry(LANGUAGES.indexName, LANGUAGES),
+ Map.entry(LANGUAGES_LOOKUP.indexName, LANGUAGES_LOOKUP),
  Map.entry(UL_LOGS.indexName, UL_LOGS),
  Map.entry(SAMPLE_DATA.indexName, SAMPLE_DATA),
  Map.entry(ALERTS.indexName, ALERTS),
@@ -130,9 +131,7 @@ public class CsvTestsDataLoader {
  Map.entry(DISTANCES.indexName, DISTANCES),
  Map.entry(ADDRESSES.indexName, ADDRESSES),
  Map.entry(BOOKS.indexName, BOOKS),
- Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT),
- // JOIN LOOKUP alias
- Map.entry(LANGUAGES.indexName + LOOKUP_INDEX_SUFFIX, LANGUAGES.withIndex(LANGUAGES.indexName + LOOKUP_INDEX_SUFFIX))
+ Map.entry(SEMANTIC_TEXT.indexName, SEMANTIC_TEXT)
  );
 
  private static final EnrichConfig LANGUAGES_ENRICH = new EnrichConfig("languages_policy", "enrich-policy-languages.json");

x-pack/plugin/esql/qa/testFixtures/src/main/resources/languages_lookup-settings.json

@@ -0,0 +1,5 @@
+{
+ "index": {
+ "mode": "lookup"
+ }
+}

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java

@@ -62,6 +62,7 @@
 import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
 import org.elasticsearch.xpack.esql.index.EsIndex;
+import org.elasticsearch.xpack.esql.index.IndexResolution;
 import org.elasticsearch.xpack.esql.parser.ParsingException;
 import org.elasticsearch.xpack.esql.plan.TableIdentifier;
 import org.elasticsearch.xpack.esql.plan.logical.Aggregate;
@@ -106,7 +107,6 @@
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
 import java.util.function.Function;
@@ -199,57 +199,42 @@ private static class ResolveTable extends ParameterizedAnalyzerRule<UnresolvedRe
 
  @Override
  protected LogicalPlan rule(UnresolvedRelation plan, AnalyzerContext context) {
- if (plan.indexMode().equals(IndexMode.LOOKUP)) {
- return hackLookupMapping(plan);
- }
- if (context.indexResolution().isValid() == false) {
- return plan.unresolvedMessage().equals(context.indexResolution().toString())
+ return resolveIndex(plan, plan.indexMode().equals(IndexMode.LOOKUP) ? context.lookupResolution() : context.indexResolution());
+ }
+
+ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexResolution) {
+ if (indexResolution.isValid() == false) {
+ return plan.unresolvedMessage().equals(indexResolution.toString())
  ? plan
  : new UnresolvedRelation(
  plan.source(),
  plan.table(),
  plan.frozen(),
  plan.metadataFields(),
  plan.indexMode(),
- context.indexResolution().toString(),
+ indexResolution.toString(),
  plan.commandName()
  );
  }
  TableIdentifier table = plan.table();
- if (context.indexResolution().matches(table.index()) == false) {
+ if (indexResolution.matches(table.index()) == false) {
  // TODO: fix this (and tests), or drop check (seems SQL-inherited, where's also defective)
  new UnresolvedRelation(
  plan.source(),
  plan.table(),
  plan.frozen(),
  plan.metadataFields(),
  plan.indexMode(),
- "invalid [" + table + "] resolution to [" + context.indexResolution() + "]",
+ "invalid [" + table + "] resolution to [" + indexResolution + "]",
  plan.commandName()
  );
  }
 
- EsIndex esIndex = context.indexResolution().get();
+ EsIndex esIndex = indexResolution.get();
  var attributes = mappingAsAttributes(plan.source(), esIndex.mapping());
  attributes.addAll(plan.metadataFields());
  return new EsRelation(plan.source(), esIndex, attributes.isEmpty() ? NO_FIELDS : attributes, plan.indexMode());
  }
-
- private LogicalPlan hackLookupMapping(UnresolvedRelation plan) {
- if (plan.table().index().toLowerCase(Locale.ROOT).equals("languages_lookup")) {
- EsIndex esIndex = new EsIndex(
- "languages_lookup",
- Map.ofEntries(
- Map.entry("language_code", new EsField("language_code", DataType.LONG, Map.of(), true)),
- Map.entry("language_name", new EsField("language", DataType.KEYWORD, Map.of(), true))
- ),
- Map.of("languages_lookup", IndexMode.LOOKUP)
- );
- var attributes = mappingAsAttributes(plan.source(), esIndex.mapping());
- return new EsRelation(plan.source(), esIndex, attributes.isEmpty() ? NO_FIELDS : attributes, plan.indexMode());
- }
- return plan;
- }
  }
 
  /**

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/AnalyzerContext.java

@@ -15,5 +15,17 @@ public record AnalyzerContext(
  Configuration configuration,
  EsqlFunctionRegistry functionRegistry,
  IndexResolution indexResolution,
+ IndexResolution lookupResolution,
  EnrichResolution enrichResolution
-) {}
+) {
+ // Currently for tests only, since most do not test lookups
+ // TODO: make this even simpler, remove the enrichResolution for tests that do not require it (most tests)
+ public AnalyzerContext(
+ Configuration configuration,
+ EsqlFunctionRegistry functionRegistry,
+ IndexResolution indexResolution,
+ EnrichResolution enrichResolution
+ ) {
+ this(configuration, functionRegistry, indexResolution, IndexResolution.invalid("<none>"), enrichResolution);
+ }
+}