[Connector APIs] Enforce index prefix for managed connectors (#117778)

* [Connector APIs] Enforce manage connector index prefix * Update ConnectorIndexServiceTests to check for prefix * Update docs/changelog/117778.yaml * Fix accidental changes to muted tests * Review feedback * Review feedback --------- Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Author: jedrazb

docs/changelog/117778.yaml

@@ -0,0 +1,5 @@
+pr: 117778
+summary: "[Connector APIs] Enforce index prefix for managed connectors"
+area: Extract&Transform
+type: feature
+issues: []

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/10_connector_put.yml

@@ -152,6 +152,19 @@ setup:
  service_type: super-connector
 
 
+---
+'Create Connector - Invalid Managed Connector Index Prefix':
+ - do:
+ catch: "bad_request"
+ connector.put:
+ connector_id: test-connector-test-managed
+ body:
+ index_name: wrong-prefix-index
+ name: my-connector
+ language: pl
+ is_native: true
+ service_type: super-connector
+
 ---
 'Create Connector - Id returned as part of response':
  - do:

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/130_connector_update_index_name.yml

@@ -151,3 +151,18 @@ setup:
 
  - match: { index_name: content-search-2-test }
 
+---
+"Update Managed Connector Index Name - Bad Prefix":
+ - do:
+ connector.put:
+ connector_id: test-connector-2
+ body:
+ is_native: true
+ service_type: super-connector
+
+ - do:
+ catch: "bad_request"
+ connector.update_index_name:
+ connector_id: test-connector-2
+ body:
+ index_name: wrong-prefix-search-2-test

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/140_connector_update_native.yml

@@ -73,3 +73,43 @@ setup:
  field_1: test
  field_2: something
 
+---
+"Update Connector Native - changing connector to Elastic-managed wrong index name":
+
+ - do:
+ connector.put:
+ connector_id: test-connector-1
+ body:
+ is_native: false
+ index_name: super-connector
+
+ - do:
+ catch: "bad_request"
+ connector.update_native:
+ connector_id: test-connector-1
+ body:
+ is_native: true
+
+---
+"Update Connector Native - changing connector to Elastic-managed correct index name":
+
+ - do:
+ connector.put:
+ connector_id: test-connector-1
+ body:
+ is_native: false
+ index_name: content-super-connector
+
+ - do:
+ connector.update_native:
+ connector_id: test-connector-1
+ body:
+ is_native: true
+
+ - match: { result: updated }
+
+ - do:
+ connector.get:
+ connector_id: test-connector-1
+
+ - match: { is_native: true }

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/15_connector_post.yml

@@ -103,6 +103,18 @@ setup:
  service_type: super-connector
 
 
+---
+'Create Connector - Invalid Managed Connector Index Prefix':
+ - do:
+ catch: "bad_request"
+ connector.post:
+ body:
+ index_name: wrong-prefix-index
+ name: my-connector
+ language: pl
+ is_native: true
+ service_type: super-connector
+
 ---
 'Create Connector - Index name used by another connector':
  - do:

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java

@@ -76,6 +76,7 @@
 import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
 import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.fromXContentBytesConnectorFiltering;
 import static org.elasticsearch.xpack.application.connector.ConnectorFiltering.sortFilteringRulesByOrder;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX;
 
 /**
  * A service that manages persistent {@link Connector} configurations.
@@ -807,38 +808,71 @@ public void updateConnectorLastSyncStats(UpdateConnectorLastSyncStatsAction.Requ
  }
 
  /**
- * Updates the is_native property of a {@link Connector}. It always sets the {@link ConnectorStatus} to
- * CONFIGURED.
+ * Updates the is_native property of a {@link Connector}. It sets the {@link ConnectorStatus} to
+ * CONFIGURED when connector is in CONNECTED state to indicate that connector needs to reconnect.
  *
  * @param request The request for updating the connector's is_native property.
  * @param listener The listener for handling responses, including successful updates or errors.
  */
  public void updateConnectorNative(UpdateConnectorNativeAction.Request request, ActionListener<UpdateResponse> listener) {
  try {
  String connectorId = request.getConnectorId();
+ boolean isNative = request.isNative();
 
- final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc(
- new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
- .id(connectorId)
- .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
- .source(
- Map.of(
- Connector.IS_NATIVE_FIELD.getPreferredName(),
- request.isNative(),
- Connector.STATUS_FIELD.getPreferredName(),
- ConnectorStatus.CONFIGURED.toString()
- )
- )
+ getConnector(connectorId, listener.delegateFailure((l, connector) -> {
 
- );
- client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, updateResponse) -> {
- if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
- l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+ String indexName = getConnectorIndexNameFromSearchResult(connector);
+
+ boolean doesNotHaveContentPrefix = indexName != null && isValidManagedConnectorIndexName(indexName) == false;
+ // Ensure attached content index is prefixed correctly
+ if (isNative && doesNotHaveContentPrefix) {
+ l.onFailure(
+ new ElasticsearchStatusException(
+ "The index name ["
+ + indexName
+ + "] attached to the connector ["
+ + connectorId
+ + "] must start with the required prefix: ["
+ + MANAGED_CONNECTOR_INDEX_PREFIX
+ + "] to be Elastic-managed. Please update the attached index first to comply with this requirement.",
+ RestStatus.BAD_REQUEST
+ )
+ );
  return;
  }
- l.onResponse(updateResponse);
- }));
 
+ ConnectorStatus status = getConnectorStatusFromSearchResult(connector);
+
+ // If connector was connected already, change its status to CONFIGURED as we need to re-connect
+ boolean isConnected = status == ConnectorStatus.CONNECTED;
+ boolean isValidTransitionToConfigured = ConnectorStateMachine.isValidTransition(status, ConnectorStatus.CONFIGURED);
+ if (isConnected && isValidTransitionToConfigured) {
+ status = ConnectorStatus.CONFIGURED;
+ }
+
+ final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy(
+ WriteRequest.RefreshPolicy.IMMEDIATE
+ )
+ .doc(
+ new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
+ .id(connectorId)
+ .source(
+ Map.of(
+ Connector.IS_NATIVE_FIELD.getPreferredName(),
+ isNative,
+ Connector.STATUS_FIELD.getPreferredName(),
+ status.toString()
+ )
+ )
+ );
+ client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> {
+ if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+ ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+ return;
+ }
+ ll.onResponse(updateResponse);
+ }));
+ }));
  } catch (Exception e) {
  listener.onFailure(e);
  }
@@ -896,22 +930,45 @@ public void updateConnectorIndexName(UpdateConnectorIndexNameAction.Request requ
  return;
  }
 
- final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc(
- new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
- .id(connectorId)
- .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
- .source(new HashMap<>() {
- {
- put(Connector.INDEX_NAME_FIELD.getPreferredName(), request.getIndexName());
- }
- })
- );
- client.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (ll, updateResponse) -> {
- if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
- ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+ getConnector(connectorId, l.delegateFailure((ll, connector) -> {
+
+ Boolean isNativeConnector = getConnectorIsNativeFlagFromSearchResult(connector);
+ Boolean doesNotHaveContentPrefix = indexName != null && isValidManagedConnectorIndexName(indexName) == false;
+
+ if (isNativeConnector && doesNotHaveContentPrefix) {
+ ll.onFailure(
+ new ElasticsearchStatusException(
+ "Index attached to an Elastic-managed connector must start with the prefix: ["
+ + MANAGED_CONNECTOR_INDEX_PREFIX
+ + "]. The index name in the payload ["
+ + indexName
+ + "] doesn't comply with this requirement.",
+ RestStatus.BAD_REQUEST
+ )
+ );
  return;
  }
- ll.onResponse(updateResponse);
+
+ final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).doc(
+ new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
+ .id(connectorId)
+ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
+ .source(new HashMap<>() {
+ {
+ put(Connector.INDEX_NAME_FIELD.getPreferredName(), request.getIndexName());
+ }
+ })
+ );
+ client.update(
+ updateRequest,
+ new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (lll, updateResponse) -> {
+ if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+ lll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+ return;
+ }
+ lll.onResponse(updateResponse);
+ })
+ );
  }));
  }));
 
@@ -1064,6 +1121,18 @@ private ConnectorStatus getConnectorStatusFromSearchResult(ConnectorSearchResult
  return ConnectorStatus.connectorStatus((String) searchResult.getResultMap().get(Connector.STATUS_FIELD.getPreferredName()));
  }
 
+ private Boolean getConnectorIsNativeFlagFromSearchResult(ConnectorSearchResult searchResult) {
+ return (Boolean) searchResult.getResultMap().get(Connector.IS_NATIVE_FIELD.getPreferredName());
+ }
+
+ private String getConnectorIndexNameFromSearchResult(ConnectorSearchResult searchResult) {
+ return (String) searchResult.getResultMap().get(Connector.INDEX_NAME_FIELD.getPreferredName());
+ }
+
+ private boolean isValidManagedConnectorIndexName(String indexName) {
+ return indexName.startsWith(MANAGED_CONNECTOR_INDEX_PREFIX);
+ }
+
  @SuppressWarnings("unchecked")
  private Map<String, Object> getConnectorConfigurationFromSearchResult(ConnectorSearchResult searchResult) {
  return (Map<String, Object>) searchResult.getResultMap().get(Connector.CONFIGURATION_FIELD.getPreferredName());

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java

@@ -45,6 +45,8 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry {
  public static final String ACCESS_CONTROL_INDEX_NAME_PATTERN = ".search-acl-filter-*";
  public static final String ACCESS_CONTROL_TEMPLATE_NAME = "search-acl-filter";
 
+ public static final String MANAGED_CONNECTOR_INDEX_PREFIX = "content-";
+
  // Pipeline constants
 
  public static final String ENT_SEARCH_GENERIC_PIPELINE_NAME = "ent-search-generic-ingestion";

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/ConnectorActionRequest.java

@@ -19,6 +19,7 @@
 import java.io.IOException;
 
 import static org.elasticsearch.action.ValidateActions.addValidationError;
+import static org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry.MANAGED_CONNECTOR_INDEX_PREFIX;
 
 /**
  * Abstract base class for action requests targeting the connectors index. Implements {@link org.elasticsearch.action.IndicesRequest}
@@ -52,6 +53,32 @@ public ActionRequestValidationException validateIndexName(String indexName, Acti
  return validationException;
  }
 
+ /**
+ * Validates that the given index name starts with the required prefix for Elastic-managed connectors.
+ * If the index name does not start with the required prefix, the validation exception is updated with an error message.
+ *
+ * @param indexName The index name to validate. If null, no validation is performed.
+ * @param validationException The exception to accumulate validation errors.
+ * @return The updated or original {@code validationException} with any new validation errors added,
+ * if the index name does not start with the required prefix.
+ */
+ public ActionRequestValidationException validateManagedConnectorIndexPrefix(
+ String indexName,
+ ActionRequestValidationException validationException
+ ) {
+ if (indexName != null && indexName.startsWith(MANAGED_CONNECTOR_INDEX_PREFIX) == false) {
+ return addValidationError(
+ "Index ["
+ + indexName
+ + "] is invalid. Index attached to an Elastic-managed connector must start with the prefix: ["
+ + MANAGED_CONNECTOR_INDEX_PREFIX
+ + "]",
+ validationException
+ );
+ }
+ return validationException;
+ }
+
  @Override
  public String[] indices() {
  return new String[] { ConnectorTemplateRegistry.CONNECTOR_INDEX_NAME_PATTERN };

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PostConnectorAction.java

@@ -127,6 +127,10 @@ public ActionRequestValidationException validate() {
 
  validationException = validateIndexName(indexName, validationException);
 
+ if (Boolean.TRUE.equals(isNative)) {
+ validationException = validateManagedConnectorIndexPrefix(indexName, validationException);
+ }
+
  return validationException;
  }
 

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/action/PutConnectorAction.java

@@ -147,6 +147,10 @@ public ActionRequestValidationException validate() {
 
  validationException = validateIndexName(indexName, validationException);
 
+ if (Boolean.TRUE.equals(isNative)) {
+ validationException = validateManagedConnectorIndexPrefix(indexName, validationException);
+ }
+
  return validationException;
  }