[Connector API] Support hard deletes with new URL param in delete endpoint

docs/changelog/120200.yaml

@@ -0,0 +1,5 @@
+pr: 120200
+summary: "[Connector API] Support hard deletes with new URL param in delete endpoint"
+area: Extract&Transform
+type: feature
+issues: []

docs/reference/connector/apis/delete-connector-api.asciidoc

@@ -13,7 +13,7 @@ beta::[]
 For the most up-to-date API details, refer to {api-es}/group/endpoint-connector[Connector APIs].
 --
 
-Soft-deletes a connector and removes associated sync jobs.
+Deletes a connector and optionally removes associated sync jobs.
 
 Note: this action doesn't delete any API key, ingest pipeline or data index associated with the connector. These need to be removed manually.
 
@@ -37,6 +37,9 @@ To get started with Connector APIs, check out <<es-connectors-tutorial-api, our
 `<connector_id>`::
 (Required, string)
 
+`<hard>`::
+(Optional, boolean) If `true`, the connector doc is deleted. If `false`, connector doc is marked as deleted (soft deletion). Defaults to `false`.
+
 `delete_sync_jobs`::
 (Optional, boolean) A flag indicating if associated sync jobs should be also removed. Defaults to `false`.
 

rest-api-spec/src/main/resources/rest-api-spec/api/connector.delete.json

@@ -28,6 +28,11 @@
  ]
  },
  "params": {
+ "hard": {
+ "type": "boolean",
+ "default": false,
+ "description": "If true, the connector doc is deleted. If false, connector doc is marked as deleted (soft-deleted)."
+ },
  "delete_sync_jobs": {
  "type": "boolean",
  "default": false,

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/20_connector_list.yml

@@ -280,9 +280,7 @@ setup:
 
 ---
 "List Connectors - Soft deleted connectors / no deleted":
- - requires:
- cluster_features: ["connector_soft_deletes"]
- reason: Soft deletes were introduced in 9.0 release
+
 
  - do:
  connector.list:
@@ -293,9 +291,7 @@ setup:
 
 ---
 "List Connectors - Single soft deleted connector":
- - requires:
- cluster_features: ["connector_soft_deletes"]
- reason: Soft deletes were introduced in 9.0 release
+
 
  - do:
  connector.delete:
@@ -312,11 +308,91 @@ setup:
 
  - match: { count: 3 }
 
+
+---
+"List Connectors - Single hard deleted connector":
+
+
+ - do:
+ connector.delete:
+ connector_id: connector-a
+ hard: true
+
+ - do:
+ connector.list: {}
+
+ - match: { count: 2 }
+
+ - do:
+ connector.list:
+ include_deleted: true
+
+ - match: { count: 2 }
+
+
+---
+"List Connectors - All hard deleted connectors":
+
+
+ - do:
+ connector.delete:
+ connector_id: connector-a
+ hard: true
+
+ - do:
+ connector.delete:
+ connector_id: connector-b
+ hard: true
+
+ - do:
+ connector.delete:
+ connector_id: connector-c
+ hard: true
+
+ - do:
+ connector.list: {}
+
+ - match: { count: 0 }
+
+ - do:
+ connector.list:
+ include_deleted: true
+
+ - match: { count: 0 }
+
+---
+"List Connectors - 2 hard deleted connectors, 1 soft deleted":
+
+
+ - do:
+ connector.delete:
+ connector_id: connector-a
+ hard: false
+
+ - do:
+ connector.delete:
+ connector_id: connector-b
+ hard: true
+
+ - do:
+ connector.delete:
+ connector_id: connector-c
+ hard: true
+
+ - do:
+ connector.list: {}
+
+ - match: { count: 0 }
+
+ - do:
+ connector.list:
+ include_deleted: true
+
+ - match: { count: 1 }
+
 ---
 "List Connectors - Soft deleted connectors":
- - requires:
- cluster_features: ["connector_soft_deletes"]
- reason: Soft deletes were introduced in 9.0 release
+
 
  - do:
  connector.delete:
@@ -353,9 +429,7 @@ setup:
 
 ---
 "List Connectors - Soft deleted with from":
- - requires:
- cluster_features: ["connector_soft_deletes"]
- reason: Soft deletes were introduced in 9.0 release
+
 
  - do:
  connector.delete:
@@ -387,9 +461,7 @@ setup:
 
 ---
 "List Connector - Soft deleted with size":
- - requires:
- cluster_features: ["connector_soft_deletes"]
- reason: Soft deletes were introduced in 9.0 release
+
 
  - do:
  connector.delete:

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/30_connector_delete.yml

@@ -27,6 +27,31 @@ setup:
  connector_id: test-connector-to-delete
 
 
+---
+"Delete Connector - Hard Delete":
+ - do:
+ connector.put:
+ connector_id: test-connector-hard-delete
+ body:
+ index_name: search-2-test
+ name: my-hard-delete-connector
+ language: en
+ is_native: false
+ service_type: super-connector
+
+ - do:
+ connector.delete:
+ connector_id: test-connector-hard-delete
+ hard: true
+
+ - match: { acknowledged: true }
+
+ - do:
+ catch: "missing"
+ connector.get:
+ connector_id: test-connector-hard-delete
+ include_deleted: true
+
 ---
 "Delete Connector - deletes associated sync jobs":
 
@@ -107,12 +132,9 @@ setup:
  connector.delete:
  connector_id: test-nonexistent-connector
 
-
 ---
 "Delete Connector - Supports soft deletes":
- - requires:
- cluster_features: ["connector_soft_deletes"]
- reason: Soft deletes were introduced in 9.0 release
+
 
  - do:
  connector.delete:

x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorIndexService.java

@@ -13,6 +13,8 @@
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.DelegatingActionListener;
 import org.elasticsearch.action.DocWriteRequest;
+import org.elasticsearch.action.DocWriteResponse;
+import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -232,40 +234,71 @@ public void getConnector(String connectorId, boolean includeDeleted, ActionListe
  }
 
  /**
- * Soft deletes the {@link Connector} and optionally removes the related instances of {@link ConnectorSyncJob} in the underlying index.
+ * Deletes the {@link Connector} and optionally removes the related instances of {@link ConnectorSyncJob} in the underlying index.
  *
  * @param connectorId The id of the {@link Connector}.
+ * @param hardDelete If set to true, the {@link Connector} is permanently deleted; otherwise, it is soft-deleted.
  * @param shouldDeleteSyncJobs The flag indicating if {@link ConnectorSyncJob} should also be deleted.
  * @param listener The action listener to invoke on response/failure.
  */
- public void deleteConnector(String connectorId, boolean shouldDeleteSyncJobs, ActionListener<UpdateResponse> listener) {
+ public void deleteConnector(
+ String connectorId,
+ boolean hardDelete,
+ boolean shouldDeleteSyncJobs,
+ ActionListener<DocWriteResponse> listener
+ ) {
 
  try {
- // ensure that if connector is soft-deleted, deleting it again results in 404
- getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
- final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy(
- WriteRequest.RefreshPolicy.IMMEDIATE
- )
- .doc(
- new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
- .id(connectorId)
- .source(Map.of(Connector.IS_DELETED_FIELD.getPreferredName(), true))
- );
- clientWithOrigin.update(updateRequest, new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
- if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
- ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
- return;
- }
- if (shouldDeleteSyncJobs) {
- new ConnectorSyncJobIndexService(clientWithOrigin).deleteAllSyncJobsByConnectorId(
- connectorId,
- ll.map(r -> updateResponse)
+ if (hardDelete) {
+ final DeleteRequest deleteRequest = new DeleteRequest(CONNECTOR_INDEX_NAME).id(connectorId)
+ .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
+
+ clientWithOrigin.delete(
+ deleteRequest,
+ new DelegatingIndexNotFoundActionListener<>(connectorId, listener, (l, deleteResponse) -> {
+ if (deleteResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) {
+ l.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+ return;
+ }
+ if (shouldDeleteSyncJobs) {
+ new ConnectorSyncJobIndexService(clientWithOrigin).deleteAllSyncJobsByConnectorId(
+ connectorId,
+ l.map(r -> deleteResponse)
+ );
+ } else {
+ l.onResponse(deleteResponse);
+ }
+ })
+ );
+ } else {
+ getConnector(connectorId, false, listener.delegateFailure((l, connector) -> {
+ final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_INDEX_NAME, connectorId).setRefreshPolicy(
+ WriteRequest.RefreshPolicy.IMMEDIATE
+ )
+ .doc(
+ new IndexRequest(CONNECTOR_INDEX_NAME).opType(DocWriteRequest.OpType.INDEX)
+ .id(connectorId)
+ .source(Map.of(Connector.IS_DELETED_FIELD.getPreferredName(), true))
  );
- } else {
- ll.onResponse(updateResponse);
- }
+ clientWithOrigin.update(
+ updateRequest,
+ new DelegatingIndexNotFoundActionListener<>(connectorId, l, (ll, updateResponse) -> {
+ if (updateResponse.getResult() == UpdateResponse.Result.NOT_FOUND) {
+ ll.onFailure(new ResourceNotFoundException(connectorNotFoundErrorMsg(connectorId)));
+ return;
+ }
+ if (shouldDeleteSyncJobs) {
+ new ConnectorSyncJobIndexService(clientWithOrigin).deleteAllSyncJobsByConnectorId(
+ connectorId,
+ ll.map(r -> updateResponse)
+ );
+ } else {
+ ll.onResponse(updateResponse);
+ }
+ })
+ );
  }));
- }));
+ }
  } catch (Exception e) {
  listener.onFailure(e);
  }