ILM policies with searchable_snapshot and allocate in the cold phase will get stuck #77269
Description
Elasticsearch version (bin/elasticsearch --version): 7.14.1
Description of the problem including expected versus actual behavior:
If an ILM policy uses both searchable_snapshot and allocate in the cold phase, then the allocate action won't work right, and in fact will get wedged permanently (see workaround below for how to unstick it).
Steps to reproduce:
PUT _cluster/settings { "transient": { "indices.lifecycle.poll_interval": "30s" } } PUT _ilm/policy/test-policy { "policy": { "phases": { "hot": { "min_age": "0m", "actions": { "set_priority": { "priority": 100 } } }, "warm": { "min_age": "2m", "actions": { "set_priority": { "priority": 50 } } }, "cold": { "min_age": "4m", "actions": { "set_priority": { "priority": 100 }, "searchable_snapshot": { "snapshot_repository": "found-snapshots" }, "allocate": { "number_of_replicas": 0 } } }, "frozen": { "min_age": "6m", "actions": { "searchable_snapshot": { "snapshot_repository": "found-snapshots" } } } } } } PUT _template/test-template { "index_patterns": ["test-index*"], "settings": { "lifecycle": { "name": "test-policy" }, "number_of_shards": 1, "number_of_replicas": 1 } } POST test-index-1/_doc { "field1": "someValue2" } Once the policy gets to the allocate action & step, it'll just sit there forever:
GET test-index-1/_ilm/explain { "indices" : { "restored-test-index-1" : { "index" : "restored-test-index-1", "managed" : true, "policy" : "test-policy", "lifecycle_date_millis" : 1630689049418, "age" : "6.5m", "phase" : "cold", "phase_time_millis" : 1630689317926, "action" : "allocate", "action_time_millis" : 1630689318037, "step" : "allocate", "step_time_millis" : 1630689379467, "repository_name" : "found-snapshots", "snapshot_name" : "2021.09.03-test-index-1-test-policy-vx9cjvout8q3ahibxej20g", "phase_execution" : { "policy" : "test-policy", "phase_definition" : { "min_age" : "4m", "actions" : { "allocate" : { "number_of_replicas" : 0, "include" : { }, "exclude" : { }, "require" : { } }, "searchable_snapshot" : { "snapshot_repository" : "found-snapshots", "force_merge_index" : true }, "set_priority" : { "priority" : 100 } } }, "version" : 4, "modified_date_in_millis" : 1630689027809 } } } } Workaround:
For any stuck indices, if you manually move the stuck index to complete, then everything will pick back up.
POST /_ilm/move/restored-test-index-1 { "current_step": { "phase": "cold", "action": "allocate", "name": "allocate" }, "next_step": { "phase": "cold", "action": "complete", "name": "complete" } }