[8.x] Update grammar to rely on indexPattern instead of identifier in join target (#120784)

* Update grammar to rely on indexPattern instead of identifier in join target (#120494) This replaces identifier with indexPattern in joinTarget grammar. This change is needed to make index selection consistent between FROM and [LOOKUP] JOIN commands: * Both should use the same quotes " (currently join relies on `) * Both should allow specifying indices with - without having to quote them (not possible with join at the moment) * Both should conform to allowed index names (there are number of differences today, for example it is possible to specify test? or +test in join even though it is not a valid index name.) (cherry picked from commit 9ffe3c8)

Author: idegtiarenko

docs/changelog/120494.yaml

@@ -0,0 +1,5 @@
+pr: 120494
+summary: Update grammar to rely on `indexPattern` instead of identifier in join target
+area: ES|QL
+type: enhancement
+issues: []

x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java

@@ -538,12 +538,12 @@ record Listen(long timestamp, String songId, double duration) {
  public void testLookupJoinIndexAllowed() throws Exception {
  assumeTrue(
  "Requires LOOKUP JOIN capability",
- EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V11.capabilityName()))
+ EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
  );
 
  Response resp = runESQLCommand(
  "metadata1_read2",
- "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN `lookup-user2` ON value | KEEP x, org"
+ "ROW x = 40.0 | EVAL value = x | LOOKUP JOIN lookup-user2 ON value | KEEP x, org"
  );
  assertOK(resp);
  Map<String, Object> respMap = entityAsMap(resp);
@@ -554,7 +554,7 @@ public void testLookupJoinIndexAllowed() throws Exception {
  assertThat(respMap.get("values"), equalTo(List.of(List.of(40.0, "sales"))));
 
  // Alias, should find the index and the row
- resp = runESQLCommand("alias_user1", "ROW x = 31.0 | EVAL value = x | LOOKUP JOIN `lookup-first-alias` ON value | KEEP x, org");
+ resp = runESQLCommand("alias_user1", "ROW x = 31.0 | EVAL value = x | LOOKUP JOIN lookup-first-alias ON value | KEEP x, org");
  assertOK(resp);
  respMap = entityAsMap(resp);
  assertThat(
@@ -564,7 +564,7 @@ public void testLookupJoinIndexAllowed() throws Exception {
  assertThat(respMap.get("values"), equalTo(List.of(List.of(31.0, "sales"))));
 
  // Alias, for a row that's filtered out
- resp = runESQLCommand("alias_user1", "ROW x = 123.0 | EVAL value = x | LOOKUP JOIN `lookup-first-alias` ON value | KEEP x, org");
+ resp = runESQLCommand("alias_user1", "ROW x = 123.0 | EVAL value = x | LOOKUP JOIN lookup-first-alias ON value | KEEP x, org");
  assertOK(resp);
  respMap = entityAsMap(resp);
  assertThat(
@@ -577,12 +577,12 @@ public void testLookupJoinIndexAllowed() throws Exception {
  public void testLookupJoinIndexForbidden() throws Exception {
  assumeTrue(
  "Requires LOOKUP JOIN capability",
- EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V11.capabilityName()))
+ EsqlSpecTestCase.hasCapabilities(adminClient(), List.of(EsqlCapabilities.Cap.JOIN_LOOKUP_V12.capabilityName()))
  );
 
  var resp = expectThrows(
  ResponseException.class,
- () -> runESQLCommand("metadata1_read2", "FROM lookup-user2 | EVAL value = 10.0 | LOOKUP JOIN `lookup-user1` ON value | KEEP x")
+ () -> runESQLCommand("metadata1_read2", "FROM lookup-user2 | EVAL value = 10.0 | LOOKUP JOIN lookup-user1 ON value | KEEP x")
  );
  assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
  assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
@@ -591,22 +591,22 @@ public void testLookupJoinIndexForbidden() throws Exception {
  ResponseException.class,
  () -> runESQLCommand(
  "metadata1_read2",
- "FROM lookup-user2 | EVAL value = 10.0 | LOOKUP JOIN `lookup-first-alias` ON value | KEEP x"
+ "FROM lookup-user2 | EVAL value = 10.0 | LOOKUP JOIN lookup-first-alias ON value | KEEP x"
  )
  );
  assertThat(resp.getMessage(), containsString("Unknown index [lookup-first-alias]"));
  assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
 
  resp = expectThrows(
  ResponseException.class,
- () -> runESQLCommand("metadata1_read2", "ROW x = 10.0 | EVAL value = x | LOOKUP JOIN `lookup-user1` ON value | KEEP x")
+ () -> runESQLCommand("metadata1_read2", "ROW x = 10.0 | EVAL value = x | LOOKUP JOIN lookup-user1 ON value | KEEP x")
  );
  assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
  assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));
 
  resp = expectThrows(
  ResponseException.class,
- () -> runESQLCommand("alias_user1", "ROW x = 10.0 | EVAL value = x | LOOKUP JOIN `lookup-user1` ON value | KEEP x")
+ () -> runESQLCommand("alias_user1", "ROW x = 10.0 | EVAL value = x | LOOKUP JOIN lookup-user1 ON value | KEEP x")
  );
  assertThat(resp.getMessage(), containsString("Unknown index [lookup-user1]"));
  assertThat(resp.getResponse().getStatusLine().getStatusCode(), equalTo(HttpStatus.SC_BAD_REQUEST));

x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java

@@ -21,7 +21,7 @@
 import java.util.List;
 
 import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled;
-import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V11;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12;
 import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.ASYNC;
 
 public class MixedClusterEsqlSpecIT extends EsqlSpecTestCase {
@@ -96,7 +96,7 @@ protected boolean supportsInferenceTestService() {
 
  @Override
  protected boolean supportsIndexModeLookup() throws IOException {
- return hasCapabilities(List.of(JOIN_LOOKUP_V11.capabilityName()));
+ return hasCapabilities(List.of(JOIN_LOOKUP_V12.capabilityName()));
  }
 
  @Override

x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java

@@ -48,7 +48,7 @@
 import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2;
-import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V11;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1;
 import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST;
 import static org.elasticsearch.xpack.esql.qa.rest.EsqlSpecTestCase.Mode.SYNC;
@@ -124,7 +124,7 @@ protected void shouldSkipTest(String testName) throws IOException {
  assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName()));
  assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName()));
  assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName()));
- assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V11.capabilityName()));
+ assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName()));
  }
 
  private TestFeatureService remoteFeaturesService() throws IOException {

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RequestIndexFilteringTestCase.java

@@ -211,7 +211,7 @@ public void testIndicesDontExist() throws IOException {
  assertThat(e.getMessage(), containsString("index_not_found_exception"));
  assertThat(e.getMessage(), anyOf(containsString("no such index [foo]"), containsString("no such index [remote_cluster:foo]")));
 
- if (EsqlCapabilities.Cap.JOIN_LOOKUP_V11.isEnabled()) {
+ if (EsqlCapabilities.Cap.JOIN_LOOKUP_V12.isEnabled()) {
  e = expectThrows(
  ResponseException.class,
  () -> runEsql(timestampFilter("gte", "2020-01-01").query(from("test1") + " | LOOKUP JOIN foo ON id1"))