Write code to create datacontract CLI schema

Author: andrewrjones

customers.contract.yml

@@ -1,21 +1,25 @@
 dataset: customers
 owner: product-team@data-contracts.com
 description: All active customers of our product.
+version: 1
 
 columns:
  - name: id
- description: Unique ID in ...
+ description: Unique ID for each customer
  data_type: VARCHAR
  checks:
  - type: no_missing_values
  - type: no_duplicate_values
  - name: size
+ description: The customers t-shirt size
  data_type: VARCHAR
  checks:
  - type: invalid_count
  valid_values: ['S', 'M', 'L']
  must_be_less_than: 1
  - name: created
+ description: The timestamp at which the customer object was created
  data_type: TIMESTAMP
  - name: distance
+ description: The distance the customer is from our shop
  data_type: INTEGER

datacontractcli/.keep

@@ -1,3 +1,5 @@
+import yaml
+
 def to_atlas_schema(contract):
  hcl = f"""schema "public" {{}}
 
@@ -16,7 +18,7 @@ def to_atlas_schema(contract):
 protobuf_type_map = {
  "varchar": "string",
  "integer": "int64",
- # proto doesn't have a timestamp field, so we'll just
+ # protobuf doesn't have a timestamp field, so we'll just
  # use string for this example
  "timestamp": "string",
 }
@@ -28,4 +30,41 @@ def to_protobuf(contract):
  proto += f"\toptional {type} {column['name']} = {i};\n"
 
  proto += "}"
- return proto
+ return proto
+
+def to_datacontract_cli(contract):
+ data = dict(
+ dataContractSpecification = '0.9.3',
+ id = contract["dataset"],
+ info = dict (
+ title = contract["dataset"],
+ version = str(contract["version"]),
+ description = contract["description"],
+ owner = contract["owner"],
+ ),
+ servers = dict(
+ workshop = dict(
+ type = 'postgres',
+ host = 'localhost',
+ port = 5432,
+ database = 'postgres',
+ schema = 'public',
+ ),
+ ),
+ models = dict()
+ )
+
+ fields = dict()
+ for column in contract["columns"]:
+ fields[column["name"]] = dict(
+ description = column["description"],
+ type = column["data_type"].lower()
+ )
+
+ data["models"][contract["dataset"]] = {
+ 'fields': fields,
+ 'description': contract["description"],
+ }
+ 
+
+ return data

lib/data_contract.py

@@ -75,3 +75,9 @@
  writer.write(protobuf_schema)
  print(f'Written Protobuf schema of `{contract["dataset"]}` data contract to `{proto_file}`')
 
+datacontract_cli_schema = to_datacontract_cli(contract)
+datacontract_cli_file = 'datacontractcli/customers.yaml'
+with open(datacontract_cli_file, 'w') as writer:
+ yaml.dump(datacontract_cli_schema, writer)
+ print(f'Written datacontract CLI schema of `{contract["dataset"]}` data contract to `{datacontract_cli_file}`')
+