Add test for storage exhaustion when deleting rows (#13760)

Author: Vadim Averin

ydb/tests/olap/test_quota_exhaustion.py

@@ -1,3 +1,5 @@
+import os
+import subprocess
 import sys
 
 import ydb
@@ -23,23 +25,23 @@ def setup_class(cls):
  def teardown_class(cls):
  cls.cluster.stop()
 
- @link_test_case("#13529")
- def test(self):
+ def make_session(self):
  driver = ydb.Driver(endpoint=f'grpc://localhost:{self.cluster.nodes[1].grpc_port}', database='/Root')
  session = ydb.QuerySessionPool(driver)
  driver.wait(5, fail_fast=True)
+ return session
 
-  def create_table(table):
-  return session.execute_with_retries(f"""
+ def create_test_table(self, session, table):
+ return session.execute_with_retries(f"""
  CREATE TABLE {table} (
  k Int32 NOT NULL,
  v Uint64,
  PRIMARY KEY (k)
  ) WITH (STORE = COLUMN)
  """)
 
-  def upsert_chunk(table, chunk_id, retries=10):
-  return session.execute_with_retries(f"""
+ def upsert_test_chunk(self, session, table, chunk_id, retries=10):
+ return session.execute_with_retries(f"""
  $n = {ROWS_CHUNK_SIZE};
  $values_list = ListReplicate(42ul, $n);
  $rows_list = ListFoldMap($values_list, {chunk_id * ROWS_CHUNK_SIZE}, ($val, $i) -> ((<|k:$i, v:$val|>, $i + 1)));
@@ -48,17 +50,96 @@ def upsert_chunk(table, chunk_id, retries=10):
  SELECT * FROM AS_TABLE($rows_list);
  """, None, ydb.retries.RetrySettings(max_retries=retries))
 
- create_table('huge')
-
+ def upsert_until_overload(self, session, table):
  try:
  for i in range(ROWS_CHUNKS_COUNT):
- res = upsert_chunk('huge', i, retries=0)
- print(f"query #{i} ok, result:", res, file=sys.stderr)
+ res = self.upsert_test_chunk(session, table, i, retries=0)
+ print(f"upsert #{i} ok, result:", res, file=sys.stderr)
  except ydb.issues.Overloaded:
- print('got overload issue', file=sys.stderr)
+ print('upsert: got overload issue', file=sys.stderr)
 
+ @link_test_case("#13529")
+ def test(self):
+ """As per https://github.com/ydb-platform/ydb/issues/13529"""
+ session = self.make_session()
+
+ # Overflow the database
+ self.create_test_table(session, 'huge')
+ self.upsert_until_overload(session, 'huge')
+
+ # Cleanup
  session.execute_with_retries("""DROP TABLE huge""")
 
  # Check database health after cleanup
- create_table('small')
- upsert_chunk('small', 0)
+ self.create_test_table(session, 'small')
+ self.upsert_test_chunk(session, 'small', 0)
+
+ def delete_test_chunk(self, session, table, chunk_id, retries=10):
+ session.execute_with_retries(f"""
+ DELETE FROM {table}
+ WHERE {chunk_id * ROWS_CHUNK_SIZE} <= k AND k <= {chunk_id * ROWS_CHUNK_SIZE + ROWS_CHUNK_SIZE}
+ """, None, ydb.retries.RetrySettings(max_retries=retries))
+
+ def delete_until_overload(self, session, table):
+ for i in range(ROWS_CHUNKS_COUNT):
+ try:
+ self.delete_test_chunk(session, table, i, retries=0)
+ print(f"delete #{i} ok", file=sys.stderr)
+ except ydb.issues.Overloaded:
+ print('delete: got overload issue', file=sys.stderr)
+ return i
+
+ def ydbcli_db_schema_exec(self, node, operation_proto):
+ endpoint = f"{node.host}:{node.port}"
+ args = [
+ node.binary_path,
+ f"--server=grpc://{endpoint}",
+ "db",
+ "schema",
+ "exec",
+ operation_proto,
+ ]
+ command = subprocess.run(args, capture_output=True)
+ assert command.returncode == 0, command.stderr.decode("utf-8")
+
+ def alter_database_quotas(self, node, database_path, database_quotas):
+ alter_proto = """ModifyScheme {
+ OperationType: ESchemeOpAlterSubDomain
+ WorkingDir: "%s"
+ SubDomain {
+ Name: "%s"
+ DatabaseQuotas {
+ %s
+ }
+ }
+ }""" % (
+ os.path.dirname(database_path),
+ os.path.basename(database_path),
+ database_quotas,
+ )
+
+ self.ydbcli_db_schema_exec(node, alter_proto)
+
+ def test_delete(self):
+ """As per https://github.com/ydb-platform/ydb/issues/13653"""
+ session = self.make_session()
+
+ # Set soft and hard quotas to 6GB
+ self.alter_database_quotas(self.cluster.nodes[1], '/Root', """
+ data_size_hard_quota: 6000000000
+ data_size_soft_quota: 6000000000
+ """)
+
+ # Overflow the database
+ self.create_test_table(session, 'huge')
+ self.upsert_until_overload(session, 'huge')
+
+ # Check that deletion works at least first time
+ # self.delete_test_chunk(session, 'huge', 0)
+ # ^ uncomment after fixing https://github.com/ydb-platform/ydb/issues/13808
+
+ # Check that deletions will lead to overflow at some moment
+ i = self.delete_until_overload(session, 'huge')
+
+ # Try to wait until deletion works again (after compaction)
+ self.delete_test_chunk(session, 'huge', i)