[feat] PIP-442: Add memory limits for CommandGetTopicsOfNamespace (#24833)

Author: lhotari

conf/broker.conf

@@ -567,6 +567,30 @@ allowOverrideEntryFilters=false
 # Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic
 maxConcurrentLookupRequest=50000
 
+# Maximum heap memory for inflight topic list operations (MB).
+# Default: 100 MB (supports ~1M topic names assuming 100 bytes each)
+maxTopicListInFlightHeapMemSizeMB=100
+
+# Maximum direct memory for inflight topic list responses (MB).
+# Default: 100 MB (network buffers for serialized responses)
+maxTopicListInFlightDirectMemSizeMB=100
+
+# Timeout for acquiring heap memory permits (milliseconds).
+# Default: 25000 (25 seconds)
+maxTopicListInFlightHeapMemSizePermitsAcquireTimeoutMillis=25000
+
+# Maximum queue size for heap memory permit requests.
+# Default: 10000 (prevent unbounded queueing)
+maxTopicListInFlightHeapMemSizePermitsAcquireQueueSize=10000
+
+# Timeout for acquiring direct memory permits (milliseconds).
+# Default: 25000 (25 seconds)
+maxTopicListInFlightDirectMemSizePermitsAcquireTimeoutMillis=25000
+
+# Maximum queue size for direct memory permit requests.
+# Default: 10000 (prevent unbounded queueing)
+maxTopicListInFlightDirectMemSizePermitsAcquireQueueSize=10000
+
 # Max number of concurrent topic loading request broker allows to control number of zk-operations
 maxConcurrentTopicLoadRequest=5000
 

conf/proxy.conf

@@ -228,6 +228,30 @@ maxConcurrentInboundConnectionsPerIp=0
 # Max concurrent outbound connections. The proxy will error out requests beyond that.
 maxConcurrentLookupRequests=50000
 
+# Maximum heap memory for inflight topic list operations (MB).
+# Default: 100 MB (supports ~1M topic names assuming 100 bytes each)
+maxTopicListInFlightHeapMemSizeMB=100
+
+# Maximum direct memory for inflight topic list responses (MB).
+# Default: 100 MB (network buffers for serialized responses)
+maxTopicListInFlightDirectMemSizeMB=100
+
+# Timeout for acquiring heap memory permits (milliseconds).
+# Default: 25000 (25 seconds)
+maxTopicListInFlightHeapMemSizePermitsAcquireTimeoutMillis=25000
+
+# Maximum queue size for heap memory permit requests.
+# Default: 10000 (prevent unbounded queueing)
+maxTopicListInFlightHeapMemSizePermitsAcquireQueueSize=10000
+
+# Timeout for acquiring direct memory permits (milliseconds).
+# Default: 25000 (25 seconds)
+maxTopicListInFlightDirectMemSizePermitsAcquireTimeoutMillis=25000
+
+# Maximum queue size for direct memory permit requests.
+# Default: 10000 (prevent unbounded queueing)
+maxTopicListInFlightDirectMemSizePermitsAcquireQueueSize=10000
+
 ##### --- TLS --- #####
 
 # Deprecated - use servicePortTls and webServicePortTls instead

conf/standalone.conf

@@ -320,6 +320,30 @@ preciseDispatcherFlowControl=false
 # Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic
 maxConcurrentLookupRequest=50000
 
+# Maximum heap memory for inflight topic list operations (MB).
+# Default: 100 MB (supports ~1M topic names assuming 100 bytes each)
+maxTopicListInFlightHeapMemSizeMB=100
+
+# Maximum direct memory for inflight topic list responses (MB).
+# Default: 100 MB (network buffers for serialized responses)
+maxTopicListInFlightDirectMemSizeMB=100
+
+# Timeout for acquiring heap memory permits (milliseconds).
+# Default: 25000 (25 seconds)
+maxTopicListInFlightHeapMemSizePermitsAcquireTimeoutMillis=25000
+
+# Maximum queue size for heap memory permit requests.
+# Default: 10000 (prevent unbounded queueing)
+maxTopicListInFlightHeapMemSizePermitsAcquireQueueSize=10000
+
+# Timeout for acquiring direct memory permits (milliseconds).
+# Default: 25000 (25 seconds)
+maxTopicListInFlightDirectMemSizePermitsAcquireTimeoutMillis=25000
+
+# Maximum queue size for direct memory permit requests.
+# Default: 10000 (prevent unbounded queueing)
+maxTopicListInFlightDirectMemSizePermitsAcquireQueueSize=10000
+
 # Max number of concurrent topic loading request broker allows to control number of zk-operations
 maxConcurrentTopicLoadRequest=5000
 

pip/pip-442.md

@@ -1391,6 +1391,42 @@ The max allowed delay for delayed delivery (in milliseconds). If the broker rece
  doc = "Max number of concurrent lookup request broker allows to throttle heavy incoming lookup traffic")
  private int maxConcurrentLookupRequest = 50000;
 
+ @FieldContext(
+ category = CATEGORY_SERVER,
+ doc = "Maximum heap memory for inflight topic list operations (MB).\n"
+ + "Default: 100 MB (supports ~1M topic names assuming 100 bytes each)")
+ private int maxTopicListInFlightHeapMemSizeMB = 100;
+
+ @FieldContext(
+ category = CATEGORY_SERVER,
+ doc = "Maximum direct memory for inflight topic list responses (MB).\n"
+ + "Default: 100 MB (network buffers for serialized responses)")
+ private int maxTopicListInFlightDirectMemSizeMB = 100;
+
+ @FieldContext(
+ category = CATEGORY_SERVER,
+ doc = "Timeout for acquiring heap memory permits (milliseconds).\n"
+ + "Default: 25000 (25 seconds)")
+ private int maxTopicListInFlightHeapMemSizePermitsAcquireTimeoutMillis = 25000;
+
+ @FieldContext(
+ category = CATEGORY_SERVER,
+ doc = "Maximum queue size for heap memory permit requests.\n"
+ + "Default: 10000 (prevent unbounded queueing)")
+ private int maxTopicListInFlightHeapMemSizePermitsAcquireQueueSize = 10000;
+
+ @FieldContext(
+ category = CATEGORY_SERVER,
+ doc = "Timeout for acquiring direct memory permits (milliseconds).\n"
+ + "Default: 25000 (25 seconds)")
+ private int maxTopicListInFlightDirectMemSizePermitsAcquireTimeoutMillis = 25000;
+
+ @FieldContext(
+ category = CATEGORY_SERVER,
+ doc = "Maximum queue size for direct memory permit requests.\n"
+ + "Default: 10000 (prevent unbounded queueing)")
+ private int maxTopicListInFlightDirectMemSizePermitsAcquireQueueSize = 10000;
+
  @FieldContext(
  dynamic = true,
  category = CATEGORY_SERVER,