Skip to content

Commit 41575d7

Browse files
stu-elasticstu-elastic
authored
SIGTERM node shutdown type (#95430)
Adds the SIGTERM shutdown type which will be used for graceful shutdown in response to SIGTERM. Right now it's the same as REPLACE.
1 parent 1be141e commit 41575d7

File tree

13 files changed

+549
-446
lines changed

13 files changed

+549
-446
lines changed

‎docs/changelog/95430.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 95430
2+
summary: SIGTERM node shutdown type
3+
area: Infra/Node Lifecycle
4+
type: enhancement
5+
issues: []

‎server/src/main/java/org/elasticsearch/cluster/metadata/SingleNodeShutdownMetadata.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
public class SingleNodeShutdownMetadata implements SimpleDiffable<SingleNodeShutdownMetadata>, ToXContentObject {
3535

3636
public static final TransportVersion REPLACE_SHUTDOWN_TYPE_ADDED_VERSION = TransportVersion.V_7_16_0;
37+
public static final TransportVersion SIGTERM_ADDED_VERSION = TransportVersion.V_8_9_0;
3738

3839
public static final ParseField NODE_ID_FIELD = new ParseField("node_id");
3940
public static final ParseField TYPE_FIELD = new ParseField("type");
@@ -199,7 +200,8 @@ public TimeValue getAllocationDelay() {
199200
@Override
200201
public void writeTo(StreamOutput out) throws IOException {
201202
out.writeString(nodeId);
202-
if (out.getTransportVersion().before(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION) && this.type == SingleNodeShutdownMetadata.Type.REPLACE) {
203+
if ((out.getTransportVersion().before(REPLACE_SHUTDOWN_TYPE_ADDED_VERSION) && this.type == SingleNodeShutdownMetadata.Type.REPLACE)
204+
|| (out.getTransportVersion().before(SIGTERM_ADDED_VERSION) && this.type == Type.SIGTERM)) {
203205
out.writeEnum(SingleNodeShutdownMetadata.Type.REMOVE);
204206
} else {
205207
out.writeEnum(type);
@@ -380,7 +382,8 @@ public SingleNodeShutdownMetadata build() {
380382
public enum Type {
381383
REMOVE,
382384
RESTART,
383-
REPLACE;
385+
REPLACE,
386+
SIGTERM; // locally-initiated version of REMOVE
384387

385388
public static Type parse(String type) {
386389
if ("remove".equals(type.toLowerCase(Locale.ROOT))) {
@@ -389,6 +392,8 @@ public static Type parse(String type) {
389392
return RESTART;
390393
} else if ("replace".equals(type.toLowerCase(Locale.ROOT))) {
391394
return REPLACE;
395+
} else if ("sigterm".equals(type.toLowerCase(Locale.ROOT))) {
396+
return SIGTERM;
392397
} else {
393398
throw new IllegalArgumentException("unknown shutdown type: " + type);
394399
}

‎server/src/main/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDecider.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,12 @@ private static Decision getDecision(RoutingAllocation allocation, String nodeId)
6868
}
6969

7070
return switch (thisNodeShutdownMetadata.getType()) {
71-
case REPLACE, REMOVE -> allocation.decision(Decision.NO, NAME, "node [%s] is preparing to be removed from the cluster", nodeId);
71+
case REPLACE, REMOVE, SIGTERM -> allocation.decision(
72+
Decision.NO,
73+
NAME,
74+
"node [%s] is preparing to be removed from the cluster",
75+
nodeId
76+
);
7277
case RESTART -> allocation.decision(
7378
Decision.YES,
7479
NAME,

‎server/src/test/java/org/elasticsearch/cluster/metadata/NodesShutdownMetadataTests.java‎

Lines changed: 54 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,14 @@
88

99
package org.elasticsearch.cluster.metadata;
1010

11+
import org.elasticsearch.TransportVersion;
1112
import org.elasticsearch.Version;
1213
import org.elasticsearch.cluster.ClusterName;
1314
import org.elasticsearch.cluster.ClusterState;
1415
import org.elasticsearch.cluster.Diff;
1516
import org.elasticsearch.cluster.node.DiscoveryNode;
1617
import org.elasticsearch.cluster.node.DiscoveryNodes;
18+
import org.elasticsearch.common.io.stream.BytesStreamOutput;
1719
import org.elasticsearch.common.io.stream.Writeable;
1820
import org.elasticsearch.common.settings.Settings;
1921
import org.elasticsearch.core.TimeValue;
@@ -64,36 +66,59 @@ public void testRemoveShutdownMetadata() {
6466
}
6567

6668
public void testIsNodeShuttingDown() {
67-
NodesShutdownMetadata nodesShutdownMetadata = new NodesShutdownMetadata(
68-
Collections.singletonMap(
69-
"this_node",
70-
SingleNodeShutdownMetadata.builder()
71-
.setNodeId("this_node")
72-
.setReason("shutdown for a unit test")
73-
.setType(randomBoolean() ? SingleNodeShutdownMetadata.Type.REMOVE : SingleNodeShutdownMetadata.Type.RESTART)
74-
.setStartedAtMillis(randomNonNegativeLong())
75-
.build()
76-
)
77-
);
78-
79-
DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
80-
nodes.add(DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), "this_node"));
81-
nodes.localNodeId("this_node");
82-
nodes.masterNodeId("this_node");
83-
84-
ClusterState state = ClusterState.builder(ClusterName.DEFAULT).nodes(nodes).build();
85-
86-
state = ClusterState.builder(state)
87-
.metadata(Metadata.builder(state.metadata()).putCustom(NodesShutdownMetadata.TYPE, nodesShutdownMetadata).build())
88-
.nodes(
89-
DiscoveryNodes.builder(state.nodes())
90-
.add(new DiscoveryNode("_node_1", buildNewFakeTransportAddress(), Version.CURRENT))
91-
.build()
92-
)
93-
.build();
69+
for (SingleNodeShutdownMetadata.Type type : List.of(
70+
SingleNodeShutdownMetadata.Type.RESTART,
71+
SingleNodeShutdownMetadata.Type.REMOVE,
72+
SingleNodeShutdownMetadata.Type.SIGTERM
73+
)) {
74+
NodesShutdownMetadata nodesShutdownMetadata = new NodesShutdownMetadata(
75+
Collections.singletonMap(
76+
"this_node",
77+
SingleNodeShutdownMetadata.builder()
78+
.setNodeId("this_node")
79+
.setReason("shutdown for a unit test")
80+
.setType(type)
81+
.setStartedAtMillis(randomNonNegativeLong())
82+
.build()
83+
)
84+
);
85+
86+
DiscoveryNodes.Builder nodes = DiscoveryNodes.builder();
87+
nodes.add(DiscoveryNode.createLocal(Settings.EMPTY, buildNewFakeTransportAddress(), "this_node"));
88+
nodes.localNodeId("this_node");
89+
nodes.masterNodeId("this_node");
90+
91+
ClusterState state = ClusterState.builder(ClusterName.DEFAULT).nodes(nodes).build();
92+
93+
state = ClusterState.builder(state)
94+
.metadata(Metadata.builder(state.metadata()).putCustom(NodesShutdownMetadata.TYPE, nodesShutdownMetadata).build())
95+
.nodes(
96+
DiscoveryNodes.builder(state.nodes())
97+
.add(new DiscoveryNode("_node_1", buildNewFakeTransportAddress(), Version.CURRENT))
98+
.build()
99+
)
100+
.build();
101+
102+
assertThat(NodesShutdownMetadata.isNodeShuttingDown(state, "this_node"), equalTo(true));
103+
assertThat(NodesShutdownMetadata.isNodeShuttingDown(state, "_node_1"), equalTo(false));
104+
}
105+
}
94106

95-
assertThat(NodesShutdownMetadata.isNodeShuttingDown(state, "this_node"), equalTo(true));
96-
assertThat(NodesShutdownMetadata.isNodeShuttingDown(state, "_node_1"), equalTo(false));
107+
public void testSigtermIsRemoveInOlderVersions() throws IOException {
108+
SingleNodeShutdownMetadata metadata = SingleNodeShutdownMetadata.builder()
109+
.setNodeId("myid")
110+
.setType(SingleNodeShutdownMetadata.Type.SIGTERM)
111+
.setReason("myReason")
112+
.setStartedAtMillis(0L)
113+
.build();
114+
BytesStreamOutput out = new BytesStreamOutput();
115+
out.setTransportVersion(TransportVersion.V_8_7_1);
116+
metadata.writeTo(out);
117+
assertThat(new SingleNodeShutdownMetadata(out.bytes().streamInput()).getType(), equalTo(SingleNodeShutdownMetadata.Type.REMOVE));
118+
119+
out = new BytesStreamOutput();
120+
metadata.writeTo(out);
121+
assertThat(new SingleNodeShutdownMetadata(out.bytes().streamInput()).getType(), equalTo(SingleNodeShutdownMetadata.Type.SIGTERM));
97122
}
98123

99124
@Override

‎server/src/test/java/org/elasticsearch/cluster/routing/UnassignedInfoTests.java‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -646,17 +646,22 @@ public void testRemainingDelayCalculationsWithUnrelatedShutdowns() throws Except
646646
* Verifies that delay calculation is not impacted when the node the shard was last assigned to was registered for removal.
647647
*/
648648
public void testRemainingDelayCalculationWhenNodeIsShuttingDownForRemoval() throws Exception {
649-
String lastNodeId = "bogusNodeId";
650-
Map<String, SingleNodeShutdownMetadata> shutdowns = new HashMap<>();
651-
SingleNodeShutdownMetadata shutdown = SingleNodeShutdownMetadata.builder()
652-
.setNodeId(lastNodeId)
653-
.setReason(this.getTestName())
654-
.setStartedAtMillis(randomNonNegativeLong())
655-
.setType(SingleNodeShutdownMetadata.Type.REMOVE)
656-
.build();
657-
shutdowns.put(shutdown.getNodeId(), shutdown);
649+
for (SingleNodeShutdownMetadata.Type type : List.of(
650+
SingleNodeShutdownMetadata.Type.REMOVE,
651+
SingleNodeShutdownMetadata.Type.SIGTERM
652+
)) {
653+
String lastNodeId = "bogusNodeId";
654+
Map<String, SingleNodeShutdownMetadata> shutdowns = new HashMap<>();
655+
SingleNodeShutdownMetadata shutdown = SingleNodeShutdownMetadata.builder()
656+
.setNodeId(lastNodeId)
657+
.setReason(this.getTestName())
658+
.setStartedAtMillis(randomNonNegativeLong())
659+
.setType(type)
660+
.build();
661+
shutdowns.put(shutdown.getNodeId(), shutdown);
658662

659-
checkRemainingDelayCalculation(lastNodeId, TimeValue.timeValueNanos(10), shutdowns, TimeValue.timeValueNanos(10), false);
663+
checkRemainingDelayCalculation(lastNodeId, TimeValue.timeValueNanos(10), shutdowns, TimeValue.timeValueNanos(10), false);
664+
}
660665
}
661666

662667
/**

‎server/src/test/java/org/elasticsearch/cluster/routing/allocation/decider/NodeShutdownAllocationDeciderTests.java‎

Lines changed: 54 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import java.util.Arrays;
3939
import java.util.Collections;
4040
import java.util.HashMap;
41+
import java.util.List;
4142

4243
import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings;
4344
import static org.hamcrest.Matchers.equalTo;
@@ -78,6 +79,12 @@ public class NodeShutdownAllocationDeciderTests extends ESAllocationTestCase {
7879
)
7980
.build();
8081

82+
private static final List<SingleNodeShutdownMetadata.Type> REMOVE_SHUTDOWN_TYPES = List.of(
83+
SingleNodeShutdownMetadata.Type.REPLACE,
84+
SingleNodeShutdownMetadata.Type.REMOVE,
85+
SingleNodeShutdownMetadata.Type.SIGTERM
86+
);
87+
8188
public void testCanAllocateShardsToRestartingNode() {
8289
ClusterState state = prepareState(
8390
service.reroute(ClusterState.EMPTY_STATE, "initial state", ActionListener.noop()),
@@ -96,17 +103,16 @@ public void testCanAllocateShardsToRestartingNode() {
96103
}
97104

98105
public void testCannotAllocateShardsToRemovingNode() {
99-
ClusterState state = prepareState(
100-
service.reroute(ClusterState.EMPTY_STATE, "initial state", ActionListener.noop()),
101-
randomFrom(SingleNodeShutdownMetadata.Type.REMOVE, SingleNodeShutdownMetadata.Type.REPLACE)
102-
);
103-
RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state, null, null, 0);
104-
RoutingNode routingNode = RoutingNodesHelper.routingNode(DATA_NODE.getId(), DATA_NODE, shard);
105-
allocation.debugDecision(true);
106-
107-
Decision decision = decider.canAllocate(shard, routingNode, allocation);
108-
assertThat(decision.type(), equalTo(Decision.Type.NO));
109-
assertThat(decision.getExplanation(), equalTo("node [" + DATA_NODE.getId() + "] is preparing to be removed from the cluster"));
106+
for (SingleNodeShutdownMetadata.Type type : REMOVE_SHUTDOWN_TYPES) {
107+
ClusterState state = prepareState(service.reroute(ClusterState.EMPTY_STATE, "initial state", ActionListener.noop()), type);
108+
RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state, null, null, 0);
109+
RoutingNode routingNode = RoutingNodesHelper.routingNode(DATA_NODE.getId(), DATA_NODE, shard);
110+
allocation.debugDecision(true);
111+
112+
Decision decision = decider.canAllocate(shard, routingNode, allocation);
113+
assertThat(type.toString(), decision.type(), equalTo(Decision.Type.NO));
114+
assertThat(decision.getExplanation(), equalTo("node [" + DATA_NODE.getId() + "] is preparing to be removed from the cluster"));
115+
}
110116
}
111117

112118
public void testShardsCanRemainOnRestartingNode() {
@@ -127,17 +133,20 @@ public void testShardsCanRemainOnRestartingNode() {
127133
}
128134

129135
public void testShardsCannotRemainOnRemovingNode() {
130-
ClusterState state = prepareState(
131-
service.reroute(ClusterState.EMPTY_STATE, "initial state", ActionListener.noop()),
132-
randomFrom(SingleNodeShutdownMetadata.Type.REMOVE, SingleNodeShutdownMetadata.Type.REPLACE)
133-
);
134-
RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state, null, null, 0);
135-
RoutingNode routingNode = RoutingNodesHelper.routingNode(DATA_NODE.getId(), DATA_NODE, shard);
136-
allocation.debugDecision(true);
137-
138-
Decision decision = decider.canRemain(null, shard, routingNode, allocation);
139-
assertThat(decision.type(), equalTo(Decision.Type.NO));
140-
assertThat(decision.getExplanation(), equalTo("node [" + DATA_NODE.getId() + "] is preparing to be removed from the cluster"));
136+
for (SingleNodeShutdownMetadata.Type type : REMOVE_SHUTDOWN_TYPES) {
137+
ClusterState state = prepareState(service.reroute(ClusterState.EMPTY_STATE, "initial state", ActionListener.noop()), type);
138+
RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state, null, null, 0);
139+
RoutingNode routingNode = RoutingNodesHelper.routingNode(DATA_NODE.getId(), DATA_NODE, shard);
140+
allocation.debugDecision(true);
141+
142+
Decision decision = decider.canRemain(null, shard, routingNode, allocation);
143+
assertThat(type.toString(), decision.type(), equalTo(Decision.Type.NO));
144+
assertThat(
145+
type.toString(),
146+
decision.getExplanation(),
147+
equalTo("node [" + DATA_NODE.getId() + "] is preparing to be removed from the cluster")
148+
);
149+
}
141150
}
142151

143152
public void testCanAutoExpandToRestartingNode() {
@@ -168,31 +177,32 @@ public void testCanAutoExpandToNodeIfNoNodesShuttingDown() {
168177
}
169178

170179
public void testCanAutoExpandToNodeThatIsNotShuttingDown() {
171-
ClusterState state = prepareState(
172-
service.reroute(ClusterState.EMPTY_STATE, "initial state", ActionListener.noop()),
173-
randomFrom(SingleNodeShutdownMetadata.Type.REMOVE, SingleNodeShutdownMetadata.Type.REPLACE),
174-
"other-node-id"
175-
);
176-
177-
RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state, null, null, 0);
178-
allocation.debugDecision(true);
179-
180-
Decision decision = decider.shouldAutoExpandToNode(indexMetadata, DATA_NODE, allocation);
181-
assertThat(decision.type(), equalTo(Decision.Type.YES));
182-
assertThat(decision.getExplanation(), equalTo("this node is not shutting down"));
180+
for (SingleNodeShutdownMetadata.Type type : REMOVE_SHUTDOWN_TYPES) {
181+
ClusterState state = prepareState(
182+
service.reroute(ClusterState.EMPTY_STATE, "initial state", ActionListener.noop()),
183+
type,
184+
"other-node-id"
185+
);
186+
187+
RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state, null, null, 0);
188+
allocation.debugDecision(true);
189+
190+
Decision decision = decider.shouldAutoExpandToNode(indexMetadata, DATA_NODE, allocation);
191+
assertThat(type.toString(), decision.type(), equalTo(Decision.Type.YES));
192+
assertThat(type.toString(), decision.getExplanation(), equalTo("this node is not shutting down"));
193+
}
183194
}
184195

185196
public void testCannotAutoExpandToRemovingNode() {
186-
ClusterState state = prepareState(
187-
service.reroute(ClusterState.EMPTY_STATE, "initial state", ActionListener.noop()),
188-
randomFrom(SingleNodeShutdownMetadata.Type.REMOVE, SingleNodeShutdownMetadata.Type.REPLACE)
189-
);
190-
RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state, null, null, 0);
191-
allocation.debugDecision(true);
192-
193-
Decision decision = decider.shouldAutoExpandToNode(indexMetadata, DATA_NODE, allocation);
194-
assertThat(decision.type(), equalTo(Decision.Type.NO));
195-
assertThat(decision.getExplanation(), equalTo("node [" + DATA_NODE.getId() + "] is preparing to be removed from the cluster"));
197+
for (SingleNodeShutdownMetadata.Type type : REMOVE_SHUTDOWN_TYPES) {
198+
ClusterState state = prepareState(service.reroute(ClusterState.EMPTY_STATE, "initial state", ActionListener.noop()), type);
199+
RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state, null, null, 0);
200+
allocation.debugDecision(true);
201+
202+
Decision decision = decider.shouldAutoExpandToNode(indexMetadata, DATA_NODE, allocation);
203+
assertThat(decision.type(), equalTo(Decision.Type.NO));
204+
assertThat(decision.getExplanation(), equalTo("node [" + DATA_NODE.getId() + "] is preparing to be removed from the cluster"));
205+
}
196206
}
197207

198208
private ClusterState prepareState(ClusterState initialState, SingleNodeShutdownMetadata.Type shutdownType) {

0 commit comments

Comments
 (0)