Consolidate top level config (#326)

# Description Part of #289 Co-authored-by: Malte Sander <malte.sander.it@gmail.com>

Author: maltesander

CHANGELOG.md

@@ -8,7 +8,13 @@ All notable changes to this project will be documented in this file.
 
 - Deploy default and support custom affinities ([#319]).
 
+### Changed
+
+- `operator-rs` `0.36.0` -> `0.37.0` ([#326]).
+- [Breaking] Moved top level config option to `clusterConfig` ([#326]).
+
 [#319]: https://github.com/stackabletech/hdfs-operator/pull/319
+[#326]: https://github.com/stackabletech/hdfs-operator/pull/326
 
 ## [23.1.0] - 2023-01-23
 

Cargo.lock

@@ -24,9 +24,26 @@ spec:
  properties:
  spec:
  properties:
- autoFormatFs:
- nullable: true
- type: boolean
+ clusterConfig:
+ properties:
+ autoFormatFs:
+ nullable: true
+ type: boolean
+ dfsReplication:
+ format: uint8
+ minimum: 0.0
+ nullable: true
+ type: integer
+ vectorAggregatorConfigMapName:
+ description: Name of the Vector aggregator discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator.
+ nullable: true
+ type: string
+ zookeeperConfigMapName:
+ description: Name of the ZooKeeper discovery config map.
+ type: string
+ required:
+ - zookeeperConfigMapName
+ type: object
  dataNodes:
  nullable: true
  properties:
@@ -1408,11 +1425,6 @@ spec:
  required:
  - roleGroups
  type: object
- dfsReplication:
- format: uint8
- minimum: 0.0
- nullable: true
- type: integer
  image:
  anyOf:
  - required:
@@ -4181,16 +4193,9 @@ spec:
  required:
  - roleGroups
  type: object
- vectorAggregatorConfigMapName:
- description: Name of the Vector aggregator discovery ConfigMap. It must contain the key `ADDRESS` with the address of the Vector aggregator.
- nullable: true
- type: string
- zookeeperConfigMapName:
- description: Name of the ZooKeeper discovery config map.
- type: string
  required:
+ - clusterConfig
  - image
- - zookeeperConfigMapName
  type: object
  required:
  - spec

deploy/helm/hdfs-operator/crds/crds.yaml

@@ -7,8 +7,9 @@ spec:
  image:
  productVersion: 3.3.4
  stackableVersion: 0.3.0
- zookeeperConfigMapName: simple-hdfs-znode
- dfsReplication: 3
+ clusterConfig:
+ zookeeperConfigMapName: simple-hdfs-znode
+ dfsReplication: 1
  nameNodes:
  roleGroups:
  default:

docs/modules/hdfs/examples/getting_started/hdfs.yaml

@@ -26,7 +26,7 @@ In the custom resource you can specify the number of replicas per role group (Na
 
 * 2 NameNodes (HA)
 * 1 JournalNode
-* 1 DataNode (should match at least the `dfsReplication` factor)
+* 1 DataNode (should match at least the `clusterConfig.dfsReplication` factor)
 
 == Supported Versions
 

docs/modules/hdfs/pages/index.adoc

@@ -6,7 +6,8 @@ ConfigMap for the aggregator and by enabling the log agent:
 [source,yaml]
 ----
 spec:
- vectorAggregatorConfigMapName: vector-aggregator-discovery
+ clusterConfig:
+ vectorAggregatorConfigMapName: vector-aggregator-discovery
  nameNodes:
  config:
  logging:

docs/modules/hdfs/pages/usage-guide/logging-log-aggregation.adoc

@@ -9,7 +9,7 @@ version = "0.0.0-dev"
 publish = false
 
 [dependencies]
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.36.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.37.0" }
 
 semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }

rust/crd/Cargo.toml

@@ -57,7 +57,8 @@ spec:
  image:
  productVersion: 3.3.4
  stackableVersion: 0.2.0
- zookeeperConfigMapName: hdfs-zk
+ clusterConfig:
+ zookeeperConfigMapName: hdfs-zk
  journalNodes:
  roleGroups:
  default:
@@ -144,7 +145,8 @@ spec:
  image:
  productVersion: 3.3.4
  stackableVersion: 0.2.0
- zookeeperConfigMapName: hdfs-zk
+ clusterConfig: 
+ zookeeperConfigMapName: hdfs-zk
  journalNodes:
  roleGroups:
  default:

rust/crd/src/affinity.rs

@@ -66,14 +66,20 @@ pub enum Error {
 #[serde(rename_all = "camelCase")]
 pub struct HdfsClusterSpec {
  pub image: ProductImage,
- pub auto_format_fs: Option<bool>,
- pub dfs_replication: Option<u8>,
  #[serde(default, skip_serializing_if = "Option::is_none")]
  pub name_nodes: Option<Role<NameNodeConfigFragment>>,
  #[serde(default, skip_serializing_if = "Option::is_none")]
  pub data_nodes: Option<Role<DataNodeConfigFragment>>,
  #[serde(default, skip_serializing_if = "Option::is_none")]
  pub journal_nodes: Option<Role<JournalNodeConfigFragment>>,
+ pub cluster_config: HdfsClusterConfig,
+}
+
+#[derive(Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
+pub struct HdfsClusterConfig {
+ pub auto_format_fs: Option<bool>,
+ pub dfs_replication: Option<u8>,
  /// Name of the Vector aggregator discovery ConfigMap.
  /// It must contain the key `ADDRESS` with the address of the Vector aggregator.
  #[serde(skip_serializing_if = "Option::is_none")]
@@ -798,7 +804,7 @@ impl Configuration for NameNodeConfigFragment {
  ) -> Result<BTreeMap<String, Option<String>>, ConfigError> {
  let mut config = BTreeMap::new();
  if file == HDFS_SITE_XML {
- if let Some(replication) = &resource.spec.dfs_replication {
+ if let Some(replication) = &resource.spec.cluster_config.dfs_replication {
  config.insert(DFS_REPLICATION.to_string(), Some(replication.to_string()));
  }
  }
@@ -929,7 +935,7 @@ impl Configuration for DataNodeConfigFragment {
  ) -> Result<BTreeMap<String, Option<String>>, ConfigError> {
  let mut config = BTreeMap::new();
  if file == HDFS_SITE_XML {
- if let Some(replication) = &resource.spec.dfs_replication {
+ if let Some(replication) = &resource.spec.cluster_config.dfs_replication {
  config.insert(DFS_REPLICATION.to_string(), Some(replication.to_string()));
  }
  }
@@ -1072,7 +1078,8 @@ spec:
  image:
  productVersion: 3.3.4
  stackableVersion: 0.2.0
- zookeeperConfigMapName: hdfs-zk
+ clusterConfig:
+ zookeeperConfigMapName: hdfs-zk
  dataNodes:
  roleGroups:
  default:
@@ -1110,7 +1117,8 @@ spec:
  image:
  productVersion: 3.3.4
  stackableVersion: 0.2.0
- zookeeperConfigMapName: hdfs-zk
+ clusterConfig:
+ zookeeperConfigMapName: hdfs-zk
  dataNodes:
  config:
  resources:
@@ -1148,7 +1156,8 @@ spec:
  image:
  productVersion: 3.3.4
  stackableVersion: 0.2.0
- zookeeperConfigMapName: hdfs-zk
+ clusterConfig:
+ zookeeperConfigMapName: hdfs-zk
  dataNodes:
  roleGroups:
  default:
@@ -1181,7 +1190,8 @@ spec:
  image:
  productVersion: 3.3.4
  stackableVersion: 0.2.0
- zookeeperConfigMapName: hdfs-zk
+ clusterConfig:
+ zookeeperConfigMapName: hdfs-zk
  nameNodes:
  roleGroups:
  default:
@@ -1245,7 +1255,8 @@ spec:
  image:
  productVersion: 3.3.4
  stackableVersion: 0.2.0
- zookeeperConfigMapName: hdfs-zk
+ clusterConfig:
+ zookeeperConfigMapName: hdfs-zk
  dataNodes:
  config:
  resources:
@@ -1299,7 +1310,8 @@ spec:
  image:
  productVersion: 3.3.4
  stackableVersion: 0.2.0
- zookeeperConfigMapName: hdfs-zk
+ clusterConfig:
+ zookeeperConfigMapName: hdfs-zk
  dataNodes:
  roleGroups:
  default:

rust/crd/src/lib.rs

@@ -10,7 +10,7 @@ build = "build.rs"
 publish = false
 
 [dependencies]
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.36.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.37.0" }
 stackable-hdfs-crd = { path = "../crd" }
 stackable-hdfs-operator = { path = "../operator" }
 anyhow = "1.0"
@@ -20,7 +20,7 @@ tracing = "0.1"
 
 [build-dependencies]
 built = { version = "0.5", features = ["chrono", "git2"] }
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.36.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.37.0" }
 stackable-hdfs-crd = { path = "../crd" }
 
 [[bin]]