m) Fix: packet_in pool memory leak (litespeedtech#426)

[Reproduce] http_client POST large body request to http_server, like: http_client -H quic.test.com -s ${SERVER_ADDR}:${SERVER_PORT} -p /1KB_char.txt -M POST -P /test/quic-data/html/10KB_char.txt -o version=h3 -n 100 -r 50000000 -R 500 -w 10 -K [Notes] [RFC5116 AEAD] Section 5.1 An authentication tag with a length of 16 octets (128bits) is used. [RFC9001 QUIC-TLS] Section 5.3 These cipher suites have a 16-byte authentication tag and produce an output 16 bytes larger than their input. Co-authored-by: wangfuyu <ivanfywang@gmail.com>

Author: wangfuyu

src/liblsquic/lsquic_enc_sess_ietf.c

@@ -2243,7 +2243,16 @@ iquic_esf_decrypt_packet (enc_session_t *enc_session_p,
  size_t out_sz;
  enum dec_packin dec_packin;
  int s;
- const size_t dst_sz = packet_in->pi_data_sz;
+ /* 16Bytes: AEAD authentication tag
+ *
+ * [RFC5116 AEAD] Section 5.1
+ * An authentication tag with a length of 16 octets (128bits) is used.
+ *
+ * [RFC9001 QUIC-TLS] Section 5.3
+ * These cipher suites have a 16-byte authentication tag and
+ * produce an output 16 bytes larger than their input.
+ */
+ const size_t dst_sz = packet_in->pi_data_sz - 16;
  unsigned char new_secret[EVP_MAX_KEY_LENGTH];
  struct crypto_ctx crypto_ctx_buf;
  char secret_str[EVP_MAX_KEY_LENGTH * 2 + 1];

src/liblsquic/lsquic_engine.c

@@ -261,6 +261,7 @@ struct lsquic_engine
  unsigned n_conns;
  lsquic_time_t deadline;
  lsquic_time_t resume_sending_at;
+ lsquic_time_t mem_logged_last;
  unsigned mini_conns_count;
  struct lsquic_purga *purga;
 #if LSQUIC_CONN_STATS
@@ -960,6 +961,51 @@ destroy_conn (struct lsquic_engine *engine, struct lsquic_conn *conn,
  --engine->n_conns;
  conn->cn_flags |= LSCONN_NEVER_TICKABLE;
  conn->cn_if->ci_destroy(conn);
+
+ if (LSQ_LOG_ENABLED(LSQ_LOG_DEBUG) /* log period: 10s */
+ && ((engine->mem_logged_last + 10000000 <= now) || (engine->n_conns == 0)))
+ {
+#define MAX_MM_STAT_LOG 4096
+ unsigned cur = 0;
+ unsigned ret = 0;
+ unsigned idx = 0;
+ char mm_log[MAX_MM_STAT_LOG] = {0};
+ struct pool_stats *poolst = NULL;
+
+ engine->mem_logged_last = now;
+
+ ret = snprintf(mm_log + cur, MAX_MM_STAT_LOG - cur,
+ "%p, conns: %u, mini_conns: %u. mm_stat, used: %zu"
+ ", pool(calls-objs_all-objs_out-max-avg-var), pib",
+ engine, engine->n_conns, engine->mini_conns_count,
+ lsquic_mm_mem_used(&engine->pub.enp_mm));
+ cur += ret;
+
+ for (idx = 0; idx < MM_N_IN_BUCKETS && cur < MAX_MM_STAT_LOG; idx++)
+ {
+ poolst = &engine->pub.enp_mm.packet_in_bstats[idx];
+ ret = snprintf(mm_log + cur, MAX_MM_STAT_LOG - cur,
+ ": [%u]%u-%u-%u-%u-%u-%u", idx,
+ poolst->ps_calls, poolst->ps_objs_all, poolst->ps_objs_out,
+ poolst->ps_max, poolst->ps_max_avg, poolst->ps_max_var);
+ cur += ret;
+ }
+
+ ret = snprintf(mm_log + cur, MAX_MM_STAT_LOG - cur, ", pob");
+ cur += ret;
+
+ for (idx = 0; idx < MM_N_OUT_BUCKETS && cur < MAX_MM_STAT_LOG; idx++)
+ {
+ poolst = &engine->pub.enp_mm.packet_out_bstats[idx];
+ ret = snprintf(mm_log + cur, MAX_MM_STAT_LOG - cur,
+ ": [%u]%u-%u-%u-%u-%u-%u", idx,
+ poolst->ps_calls, poolst->ps_objs_all, poolst->ps_objs_out,
+ poolst->ps_max, poolst->ps_max_avg, poolst->ps_max_var);
+ cur += ret;
+ }
+
+ LSQ_DEBUG("%s", mm_log);
+ }
 }
 
 

src/liblsquic/lsquic_mm.c

@@ -192,21 +192,15 @@ void
 lsquic_mm_put_packet_in (struct lsquic_mm *mm,
  struct lsquic_packet_in *packet_in)
 {
-#if LSQUIC_USE_POOLS
- unsigned idx;
- struct packet_in_buf *pib;
-
  assert(0 == packet_in->pi_refcnt);
  if (packet_in->pi_flags & PI_OWN_DATA)
  {
- pib = (struct packet_in_buf *) packet_in->pi_data;
- idx = packet_in_index(packet_in->pi_data_sz);
- SLIST_INSERT_HEAD(&mm->packet_in_bufs[idx], pib, next_pib);
+ lsquic_mm_put_packet_in_buf(mm, packet_in->pi_data, packet_in->pi_data_sz);
  }
+
+#if LSQUIC_USE_POOLS
  TAILQ_INSERT_HEAD(&mm->free_packets_in, packet_in, pi_next);
 #else
- if (packet_in->pi_flags & PI_OWN_DATA)
- free(packet_in->pi_data);
  lsquic_malo_put(packet_in);
 #endif
 }
@@ -370,6 +364,42 @@ maybe_shrink_packet_out_bufs (struct lsquic_mm *mm, unsigned idx)
 #endif
 
 
+/* If average maximum falls under 1/4 of all objects allocated, release
+ * half of the objects allocated.
+ */
+static void
+maybe_shrink_packet_in_bufs (struct lsquic_mm *mm, unsigned idx)
+{
+ struct pool_stats *poolst;
+ struct packet_in_buf *pib;
+ unsigned n_to_leave;
+
+ poolst = &mm->packet_in_bstats[idx];
+ if (poolst->ps_max_avg * 4 < poolst->ps_objs_all)
+ {
+ n_to_leave = poolst->ps_objs_all / 2;
+ while (poolst->ps_objs_all > n_to_leave
+ && (pib = SLIST_FIRST(&mm->packet_in_bufs[idx])))
+ {
+ SLIST_REMOVE_HEAD(&mm->packet_in_bufs[idx], next_pib);
+ free(pib);
+ --poolst->ps_objs_all;
+ }
+#if LSQUIC_LOG_POOL_STATS
+ LSQ_DEBUG("pib pool #%u; max avg %u; shrank from %u to %u objs",
+ idx, poolst->ps_max_avg, n_to_leave * 2, poolst->ps_objs_all);
+#endif
+ }
+#if LSQUIC_LOG_POOL_STATS
+ else
+ {
+ LSQ_DEBUG("pib pool #%u; max avg %u; objs: %u; won't shrink",
+ idx, poolst->ps_max_avg, poolst->ps_objs_all);
+ }
+#endif
+}
+
+
 void
 lsquic_mm_put_packet_out (struct lsquic_mm *mm,
  struct lsquic_packet_out *packet_out)
@@ -456,9 +486,25 @@ lsquic_mm_get_packet_in_buf (struct lsquic_mm *mm, size_t size)
  pib = SLIST_FIRST(&mm->packet_in_bufs[idx]);
  fiu_do_on("mm/packet_in_buf", FAIL_NOMEM);
  if (pib)
+ {
  SLIST_REMOVE_HEAD(&mm->packet_in_bufs[idx], next_pib);
+ poolst_allocated(&mm->packet_in_bstats[idx], 0);
+ }
  else
+ {
  pib = malloc(packet_in_sizes[idx]);
+ if (!pib)
+ {
+ return NULL;
+ }
+
+ poolst_allocated(&mm->packet_in_bstats[idx], 1);
+ }
+
+ if (poolst_has_new_sample(&mm->packet_in_bstats[idx]))
+ {
+ maybe_shrink_packet_in_bufs(mm, idx);
+ }
 #else
  pib = malloc(size);
 #endif
@@ -476,6 +522,12 @@ lsquic_mm_put_packet_in_buf (struct lsquic_mm *mm, void *mem, size_t size)
  pib = (struct packet_in_buf *) mem;
  idx = packet_in_index(size);
  SLIST_INSERT_HEAD(&mm->packet_in_bufs[idx], pib, next_pib);
+
+ poolst_freed(&mm->packet_in_bstats[idx]);
+ if (poolst_has_new_sample(&mm->packet_in_bstats[idx]))
+ {
+ maybe_shrink_packet_in_bufs(mm, idx);
+ }
 #else
  free(mem);
 #endif

src/liblsquic/lsquic_mm.h

@@ -44,6 +44,7 @@ struct lsquic_mm {
  TAILQ_HEAD(, lsquic_packet_in) free_packets_in;
  SLIST_HEAD(, packet_out_buf) packet_out_bufs[MM_N_OUT_BUCKETS];
  struct pool_stats packet_out_bstats[MM_N_OUT_BUCKETS];
+ struct pool_stats packet_in_bstats[MM_N_IN_BUCKETS];
  SLIST_HEAD(, packet_in_buf) packet_in_bufs[MM_N_IN_BUCKETS];
  SLIST_HEAD(, four_k_page) four_k_pages;
  SLIST_HEAD(, sixteen_k_page) sixteen_k_pages;