ydbd_slice: add support for using raw config.yaml (#15094)

Author: melazyk

ydb/tools/cfg/base.py

@@ -622,6 +622,10 @@ def domains(self):
  )
  return domains
 
+ @domains.setter
+ def domains(self, values):
+ self.__cluster_description["domains"] = values
+
  @property
  def domains_config(self):
  domains_config_dict = self.__cluster_description.get("domains_config", {})

ydb/tools/ydbd_slice/__init__.py

@@ -13,10 +13,10 @@
 from urllib3.exceptions import HTTPWarning
 
 from ydb.tools.cfg.walle import NopHostsInformationProvider
-from ydb.tools.ydbd_slice import nodes, handlers, cluster_description
+from ydb.tools.ydbd_slice import nodes, handlers, cluster_description, yaml_configurator
 from ydb.tools.ydbd_slice.kube import handlers as kube_handlers, docker
 
-warnings.filterwarnings("ignore", category=DeprecationWarning)
+# warnings.filterwarnings("ignore", category=DeprecationWarning)
 warnings.filterwarnings("ignore", category=HTTPWarning)
 
 
@@ -40,6 +40,9 @@
 \033[95msample-config\033[94m - get sample configuration for cluster:
  %(prog)s sample-config --cluster-type=block-4-2-8-nodes --output-file=cluster.yaml
 
+\033[95mdynconfig-generator\033[94m - generate simple dynamic configuration for cluster:
+ %(prog)s dynconfig-generator --yaml-config=config.yaml --output-file=dynconfig.yaml
+
 \033[95minstall\033[94m - full install process from scratch:
  %(prog)s install cluster.yaml --arcadia
 
@@ -254,9 +257,9 @@ def handler(signum, frame):
  raise Terminate(signum, frame)
 
 
-def safe_load_cluster_details(cluster_yaml, walle_provider):
+def safe_load_cluster_details(cluster_yaml, walle_provider, validator=None):
  try:
- cluster_details = cluster_description.ClusterDetails(cluster_yaml, walle_provider)
+ cluster_details = cluster_description.ClusterDetails(cluster_yaml, walle_provider, validator=validator)
  except IOError as io_err:
  print('', file=sys.stderr)
  print("unable to open YAML params as a file, check args", file=sys.stderr)
@@ -310,8 +313,7 @@ def deduce_components_from_args(args, cluster_details):
  return result
 
 
-def deduce_nodes_from_args(args, walle_provider, ssh_user, ssh_key_path):
- cluster_hosts = safe_load_cluster_details(args.cluster, walle_provider).hosts_names
+def deduce_nodes_from_args(args, cluster_hosts, ssh_user, ssh_key_path):
  result = cluster_hosts
 
  if args.nodes is not None:
@@ -547,6 +549,19 @@ def databases_config_path_args():
  return args
 
 
+def yaml_config_path_args():
+ args = argparse.ArgumentParser(add_help=False)
+ args.add_argument(
+ "--yaml-config",
+ metavar="YAML_CONFIG",
+ default="",
+ required=False,
+ help="Path to file with config.yaml configuration",
+ )
+
+ return args
+
+
 def cluster_type_args():
  args = argparse.ArgumentParser(add_help=False)
  available_erasure_types = [
@@ -628,38 +643,64 @@ def dispatch_run(func, args, walle_provider, need_confirmation=False):
 
  logger.debug("run func '%s' with cmd args is '%s'", func.__name__, args)
 
- cluster_details = safe_load_cluster_details(args.cluster, walle_provider)
- components = deduce_components_from_args(args, cluster_details)
-
- nodes = deduce_nodes_from_args(args, walle_provider, args.ssh_user, args.ssh_key_path)
-
  temp_dir = deduce_temp_dir_from_args(args)
+ kikimr_bin, kikimr_compressed_bin = deduce_kikimr_bin_from_args(args)
  clear_tmp = not args.dry_run and args.temp_dir is None
 
- kikimr_bin, kikimr_compressed_bin = deduce_kikimr_bin_from_args(args)
+ if args.yaml_config:
+ configurator = yaml_configurator.YamlConfigurator(
+ args.cluster,
+ temp_dir,
+ kikimr_bin,
+ kikimr_compressed_bin,
+ args.yaml_config
+ )
+ cluster_details = configurator.cluster_description
+ else:
+ cluster_details = safe_load_cluster_details(args.cluster, walle_provider)
+ configurator = cluster_description.Configurator(
+ cluster_details,
+ out_dir=temp_dir,
+ kikimr_bin=kikimr_bin,
+ kikimr_compressed_bin=kikimr_compressed_bin,
+ walle_provider=walle_provider
+ )
 
- configurator = cluster_description.Configurator(
- cluster_details,
- out_dir=temp_dir,
- kikimr_bin=kikimr_bin,
- kikimr_compressed_bin=kikimr_compressed_bin,
- walle_provider=walle_provider
- )
+ components = deduce_components_from_args(args, cluster_details)
 
  v = vars(args)
  clear_logs = v.get('clear_logs')
  yav_version = v.get('yav_version')
+
+ nodes = deduce_nodes_from_args(args, configurator.hosts_names, args.ssh_user, args.ssh_key_path)
  slice = handlers.Slice(
  components,
  nodes,
  cluster_details,
- configurator,
+ kikimr_bin,
+ kikimr_compressed_bin,
  clear_logs,
  yav_version,
  walle_provider,
+ configurator,
  )
  func(slice)
 
+ # used only for configurator and will be removed soon
+ save_raw_cfg = v.get('save_raw_cfg')
+ if save_raw_cfg and configurator:
+ logger.debug("save raw cfg to '%s'", save_raw_cfg)
+ for root, dirs, files in os.walk(temp_dir):
+ for dir in dirs:
+ os.makedirs(os.path.join(save_raw_cfg, dir), 0o755, exist_ok=True)
+
+ for file in files:
+ src = os.path.join(root, file)
+ dst = os.path.join(save_raw_cfg, os.path.relpath(src, temp_dir))
+ with open(src, 'r') as src_f:
+ with open(dst, 'w') as dst_f:
+ dst_f.write(src_f.read())
+
  if clear_tmp:
  logger.debug("remove temp dirs '%s'", temp_dir)
  # shutil.rmtree(temp_dir)
@@ -699,6 +740,7 @@ def _run(args):
  parents=[
  direct_nodes_args(),
  cluster_description_args(),
+ yaml_config_path_args(),
  binaries_args(),
  component_args(),
  log_args(),
@@ -709,6 +751,13 @@ def _run(args):
  description="Full installation of the cluster from scratch. "
  "You can use --hosts to specify particular hosts. But it is tricky.",
  )
+ mode.add_argument(
+ "--save-raw-cfg",
+ metavar="DIR",
+ required=False,
+ default="",
+ help="Directory to save all static configuration files generated by configuration.create_static_cfg and configuration.create_dynamic_cfg",
+ )
  mode.set_defaults(handler=_run)
 
 
@@ -722,6 +771,7 @@ def _run(args):
  parents=[
  direct_nodes_args(),
  cluster_description_args(),
+ yaml_config_path_args(),
  binaries_args(),
  component_args(),
  log_args(),
@@ -759,7 +809,14 @@ def _run(args):
 
  mode = modes.add_parser(
  "stop",
- parents=[direct_nodes_args(), cluster_description_args(), binaries_args(), component_args(), ssh_args()],
+ parents=[
+ direct_nodes_args(),
+ cluster_description_args(),
+ yaml_config_path_args(),
+ binaries_args(),
+ component_args(),
+ ssh_args()
+ ],
  description="Stop ydbd static instances at the nodes. "
  "If option components specified, try to stop particular component. "
  "Use --hosts to specify particular hosts."
@@ -773,7 +830,14 @@ def _run(args):
 
  mode = modes.add_parser(
  "start",
- parents=[direct_nodes_args(), cluster_description_args(), binaries_args(), component_args(), ssh_args()],
+ parents=[
+ direct_nodes_args(),
+ cluster_description_args(),
+ yaml_config_path_args(),
+ binaries_args(),
+ component_args(),
+ ssh_args()
+ ],
  description="Start all ydbd instances at the nodes. "
  "If option components specified, try to start particular component. "
  "Otherwise only kikimr-multi-all will be started. "
@@ -791,6 +855,7 @@ def _run(args):
  parents=[
  direct_nodes_args(),
  cluster_description_args(),
+ yaml_config_path_args(),
  binaries_args(),
  component_args(),
  ssh_args(),
@@ -812,6 +877,7 @@ def _run(args):
  parents=[
  direct_nodes_args(),
  cluster_description_args(),
+ yaml_config_path_args(),
  binaries_args(),
  component_args(),
  ssh_args(),
@@ -861,6 +927,24 @@ def _run(args):
  mode.set_defaults(handler=_run)
 
 
+def add_dynconfig_generator(modes):
+ def _run(args):
+ if args.yaml_config:
+ yaml_config = yaml_configurator.YamlConfig(args.yaml_config)
+
+ if args.output_file is not None and args.output_file:
+ with open(args.output_file, "w") as output:
+ output.write(yaml_config.dynamic_simple)
+
+ mode = modes.add_parser(
+ "dynconfig-generator",
+ parents=[yaml_config_path_args(), output_file()],
+ description="Generate a minimalistic dynconfig.yaml for the provided config.yaml"
+ )
+
+ mode.set_defaults(handler=_run)
+
+
 #
 # docker and kube scenarios
 def build_docker_image(build_args, docker_package, build_ydbd, image, force_rebuild):
@@ -1413,6 +1497,7 @@ def main(walle_provider=None):
  add_format_mode(modes, walle_provider)
  add_explain_mode(modes, walle_provider)
  add_sample_config_mode(modes)
+ add_dynconfig_generator(modes)
 
  add_docker_build_mode(modes)
  add_docker_push_mode(modes)
@@ -1429,8 +1514,32 @@ def main(walle_provider=None):
 
  args = parser.parse_args()
  logging.root.setLevel(args.log_level.upper())
- args.handler(args)
 
+ if not hasattr(args, 'handler'):
+ parser.print_help()
+ return
+
+ if not args.yaml_config:
+ warnings.warn(
+ '''
+Using cluster.yaml for cluster configuration is deprecated.
+Only the 'domains' section should be filled with database and slot configurations.
+The config.yaml should be passed as a raw file through the --yaml-config.
+
+Example:
+ ydbd_slice install cluster.yaml all --binary /path/to/ydbd --yaml-config /path/to/config.yaml
+
+To save the resulting configuration files from an old cluster.yaml, use the --save-raw-cfg option.
+
+Example:
+ ydbd_slice install cluster.yaml all --binary /path/to/ydbd --save-raw-cfg /path/to/save
+
+The resulting configuration files will be saved in the /path/to/save directory. You can find config.yaml in the /path/to/save/kikimr-static directory.
+''',
+ DeprecationWarning
+ )
+
+ args.handler(args)
  except KeyboardInterrupt:
  sys.exit('\nStopped by KeyboardInterrupt.')
  except Terminate:

ydb/tools/ydbd_slice/cluster_description.py

@@ -27,15 +27,15 @@ class ClusterDetails(ClusterDetailsProvider):
  SLOTS_PORTS_START = 31000
  PORTS_SHIFT = 10
 
- def __init__(self, cluster_description_path, walle_provider):
+ def __init__(self, cluster_description_path, walle_provider, validator=None):
  self.__template = None
  self.__details = None
  self.__databases = None
  self.__dynamic_slots = None
  self._cluster_description_file = cluster_description_path
  self._walle_provider = walle_provider
 
- super(ClusterDetails, self).__init__(self.template, self._walle_provider, use_new_style_cfg=True)
+ super(ClusterDetails, self).__init__(self.template, self._walle_provider, validator=validator, use_new_style_cfg=True)
 
  @property
  def template(self):
@@ -132,6 +132,10 @@ def template(self):
  def detail(self):
  return self.__cluster_details
 
+ @property
+ def hosts_names(self):
+ return self.detail.hosts_names
+
  @staticmethod
  def _generate_fake_keys():
  content = 'Keys {\n'
@@ -140,6 +144,7 @@ def _generate_fake_keys():
  content += ' Id: "fake-secret"\n'
  content += ' Version: 1\n'
  content += '}\n'
+
  return content
 
  @staticmethod