Deprecate Experimental ML command

detection_rules/eswrap.py

@@ -421,9 +421,3 @@ def index_repo(ctx: click.Context, query, from_file, save_files):
  bulk_upload_docs, importable_rules_docs = ctx.invoke(generate_rules_index, query=query, save_files=save_files)
 
  es_client.bulk(bulk_upload_docs)
-
-
-@es_group.group('experimental')
-def es_experimental():
- """[Experimental] helper commands for integrating with Elasticsearch."""
- click.secho('\n* experimental commands are use at your own risk and may change without warning *\n')

detection_rules/ml.py

@@ -16,13 +16,10 @@
 import elasticsearch
 import json
 import requests
-from eql.table import Table
 from elasticsearch import Elasticsearch
 from elasticsearch.client import IngestClient, LicenseClient, MlClient
 
-from .eswrap import es_experimental
 from .ghwrap import ManifestManager, ReleaseManifest
-from .misc import client_error
 from .schemas import definitions
 from .utils import get_path, unzip_to_dict
 
@@ -271,186 +268,3 @@ def get_ml_model_manifests_by_model_id(repo: str = 'elastic/detection-rules') ->
  break
 
  return model_manifests
-
-
-@es_experimental.group('ml')
-def ml_group():
- """Experimental machine learning commands."""
- click.secho('\n***** Deprecation Warning *****\n', fg='yellow', err=True)
- click.secho('\n* The experiment "ml" command(s) are deprecated and will be removed in a future release. *\n',
- fg='yellow', err=True)
- click.secho('\n* Command Removal Timeframe: May 1, 2025 *\n', fg='yellow', err=True)
-
-
-@ml_group.command('check-files')
-@click.pass_context
-def check_files(ctx):
- """Check ML model files on an elasticsearch instance."""
- files = MachineLearningClient.get_all_ml_files(ctx.obj['es'])
-
- results = []
- for file_type, data in files.items():
- if file_type == 'model':
- continue
- for name in list(data):
- results.append({'file_type': file_type, 'name': name})
-
- for model_name, model in files['model'].items():
- results.append({'file_type': 'model', 'name': model_name, 'related_release': model['release'].tag_name})
-
- fields = ['file_type', 'name', 'related_release']
- table = Table.from_list(fields, results)
- click.echo(table)
- return files
-
-
-@ml_group.command('remove-model')
-@click.argument('model-id', required=False)
-@click.pass_context
-def remove_model(ctx: click.Context, model_id):
- """Remove ML model files."""
- es_client = MlClient(ctx.obj['es'])
- model_ids = MachineLearningClient.get_existing_model_ids(ctx.obj['es'])
-
- if not model_id:
- model_id = click.prompt('Model ID to remove', type=click.Choice(model_ids))
-
- try:
- result = es_client.delete_trained_model(model_id)
- except elasticsearch.ConflictError as e:
- click.echo(f'{e}: try running `remove-scripts-pipelines` first')
- ctx.exit(1)
-
- table = Table.from_list(['model_id', 'status'], [{'model_id': model_id, 'status': result}])
- click.echo(table)
- return result
-
-
-@ml_group.command('remove-scripts-pipelines')
-@click.option('--dga', is_flag=True)
-@click.option('--problemchild', is_flag=True)
-@click.pass_context
-def remove_scripts_pipelines(ctx: click.Context, **ml_types):
- """Remove ML scripts and pipeline files."""
- selected_types = [k for k, v in ml_types.items() if v]
- assert selected_types, f'Specify ML types to remove: {list(ml_types)}'
- status = MachineLearningClient.remove_ml_scripts_pipelines(es_client=ctx.obj['es'], ml_type=selected_types)
-
- results = []
- for file_type, response in status.items():
- for name, result in response.items():
- results.append({'file_type': file_type, 'name': name, 'status': result})
-
- fields = ['file_type', 'name', 'status']
- table = Table.from_list(fields, results)
- click.echo(table)
- return status
-
-
-@ml_group.command('setup')
-@click.option('--model-tag', '-t',
- help='Release tag for model files staged in detection-rules (required to download files)')
-@click.option('--repo', '-r', default='elastic/detection-rules',
- help='GitHub repository hosting the model file releases (owner/repo)')
-@click.option('--model-dir', '-d', type=click.Path(exists=True, file_okay=False),
- help='Directory containing local model files')
-@click.pass_context
-def setup_bundle(ctx, model_tag, repo, model_dir):
- """Upload ML model and dependencies to enrich data."""
- es_client: Elasticsearch = ctx.obj['es']
-
- if model_tag:
- dga_client = MachineLearningClient.from_release(es_client=es_client, release_tag=model_tag, repo=repo)
- elif model_dir:
- dga_client = MachineLearningClient.from_directory(es_client=es_client, directory=model_dir)
- else:
- return client_error('model-tag or model-dir required to download model files')
-
- dga_client.verify_license()
- status = dga_client.setup()
-
- results = []
- for file_type, response in status.items():
- for name, result in response.items():
- if file_type == 'model':
- status = 'success' if result.get('create_time') else 'potential_failure'
- results.append({'file_type': file_type, 'name': name, 'status': status})
- continue
- results.append({'file_type': file_type, 'name': name, 'status': result})
-
- fields = ['file_type', 'name', 'status']
- table = Table.from_list(fields, results)
- click.echo(table)
-
- click.echo('Associated rules and jobs can be found under ML-experimental-detections releases in the repo')
- click.echo('To upload rules, run: kibana import-rules -f <ml-rule.toml>')
- click.echo('To upload ML jobs, run: es experimental upload-ml-job <ml-job.json>')
-
-
-@ml_group.command('upload-job')
-@click.argument('job-file', type=click.Path(exists=True, dir_okay=False))
-@click.option('--overwrite', '-o', is_flag=True, help='Overwrite job if exists by name')
-@click.pass_context
-def upload_job(ctx: click.Context, job_file, overwrite):
- """Upload experimental ML jobs."""
- es_client: Elasticsearch = ctx.obj['es']
- ml_client = MlClient(es_client)
-
- with open(job_file, 'r') as f:
- job = json.load(f)
-
- def safe_upload(func):
- try:
- func(name, body)
- except (elasticsearch.ConflictError, elasticsearch.RequestError) as err:
- if isinstance(err, elasticsearch.RequestError) and err.error != 'resource_already_exists_exception':
- client_error(str(err), err, ctx=ctx)
-
- if overwrite:
- ctx.invoke(delete_job, job_name=name, job_type=job_type)
- func(name, body)
- else:
- client_error(str(err), err, ctx=ctx)
-
- try:
- job_type = job['type']
- name = job['name']
- body = job['body']
-
- if job_type == 'anomaly_detection':
- safe_upload(ml_client.put_job)
- elif job_type == 'data_frame_analytic':
- safe_upload(ml_client.put_data_frame_analytics)
- elif job_type == 'datafeed':
- safe_upload(ml_client.put_datafeed)
- else:
- client_error(f'Unknown ML job type: {job_type}')
-
- click.echo(f'Uploaded {job_type} job: {name}')
- except KeyError as e:
- client_error(f'{job_file} missing required info: {e}')
-
-
-@ml_group.command('delete-job')
-@click.argument('job-name')
-@click.argument('job-type')
-@click.pass_context
-def delete_job(ctx: click.Context, job_name, job_type, verbose=True):
- """Remove experimental ML jobs."""
- es_client: Elasticsearch = ctx.obj['es']
- ml_client = MlClient(es_client)
-
- try:
- if job_type == 'anomaly_detection':
- ml_client.delete_job(job_name)
- elif job_type == 'data_frame_analytic':
- ml_client.delete_data_frame_analytics(job_name)
- elif job_type == 'datafeed':
- ml_client.delete_datafeed(job_name)
- else:
- client_error(f'Unknown ML job type: {job_type}')
- except (elasticsearch.NotFoundError, elasticsearch.ConflictError) as e:
- client_error(str(e), e, ctx=ctx)
-
- if verbose:
- click.echo(f'Deleted {job_type} job: {job_name}')

docs-dev/experimental-machine-learning/experimental-detections.md

@@ -1,3 +1,4 @@
+
 # Experimental ML Jobs and Rules
 
 The ingest pipeline enriches process events by adding additional fields, which are used to power several rules. 
@@ -16,13 +17,3 @@ Earlier releases stored the rules in toml format. These can be uploaded using th
 [7.12 branch](https://github.com/elastic/detection-rules/tree/7.12) CLI using the 
 [kibana import-rules](../../CLI.md#uploading-rules-to-kibana) command
 
-### Uploading ML Jobs and Datafeeds
-
-Unzip the release bundle and then run `python -m detection_rules es <args> experimental ml upload-job <ml_job.json>`
-
-To delete a job/datafeed, run `python -m detection_rules es <args> experimental ml delete-job <job-name> <job-type>`
-
-The CLI automatically identifies whether the provided input file is an ML job or datafeed.
-
-Take note of any errors as the jobs and datafeeds may have dependencies on each other which may require stopping and/or removing
-referenced jobs/datafeeds first.

docs-dev/experimental-machine-learning/readme.md

@@ -1,3 +1,4 @@
+
 # Experimental machine learning
 
 This repo contains some additional information and files to use experimental[*](#what-does-experimental-mean-in-this-context) machine learning features and detections
@@ -22,82 +23,6 @@ There are separate [releases](https://github.com/elastic/detection-rules/release
 Releases will use the tag `ML-TYPE-YYYMMDD-N`, which will be needed for uploading the model using the CLI.
 
 
-## CLI
-
-Support commands can be found under `python -m detection_rules es <es args> experimental ml -h`
-
-```console
-Elasticsearch client:
-Options:
- -et, --timeout INTEGER Timeout for elasticsearch client
- -ep, --es-password TEXT
- -eu, --es-user TEXT
- --elasticsearch-url TEXT
- --cloud-id TEXT
-
-
-* experimental commands are use at your own risk and may change without warning *
-
-Usage: detection_rules es experimental ml [OPTIONS] COMMAND [ARGS]...
-
- Experimental machine learning commands.
-
-Options:
- -h, --help Show this message and exit.
-
-Commands:
- check-files Check ML model files on an elasticsearch...
- delete-job Remove experimental ML jobs.
- remove-model Remove ML model files.
- remove-scripts-pipelines Remove ML scripts and pipeline files.
- setup Upload ML model and dependencies to enrich data.
- upload-job Upload experimental ML jobs.
-```
-
-## Managing a model and dependencies using the CLI
-
-### Installing
-
-```console
-python -m detection_rules es experimental ml setup -h
-
-Elasticsearch client:
-Options:
- -et, --timeout INTEGER Timeout for elasticsearch client
- -ep, --es-password TEXT
- -eu, --es-user TEXT
- --cloud-id TEXT
- --elasticsearch-url TEXT
-
-
-* experimental commands are use at your own risk and may change without warning *
-
-Usage: detection_rules es experimental ml setup [OPTIONS]
-
- Upload ML model and dependencies to enrich data.
-
-Options:
- -t, --model-tag TEXT Release tag for model files staged in detection-
- rules (required to download files)
- -r, --repo TEXT GitHub repository hosting the model file releases
- (owner/repo)
- -d, --model-dir DIRECTORY Directory containing local model files
- --overwrite Overwrite all files if already in the stack
- -h, --help Show this message and exit.
-
-```
-
-### Removing
-
-To remove the ML bundle, you will need to remove the pipelines and scripts first and then the model.
-
-You can do this by running:
-* `python -m detection_rules es experimental ml remove-pipeline-scripts --dga --problemchild`
-* `python -m detection_rules es experimental ml remove-model <model-id>`
-
-
-----
-
 ##### What does experimental mean in this context?
 
 Experimental model bundles (models, scripts, and pipelines), rules, and jobs are components which are currently in 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "detection_rules"
-version = "1.1.7"
+version = "1.2.0"
 description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine."
 readme = "README.md"
 requires-python = ">=3.12"