WIP Add metadata propagation

cwltool/builder.py

@@ -249,7 +249,7 @@ def bind_input(self,
  st = copy.deepcopy(schema["type"])
  if binding and "inputBinding" not in st and st["type"] == "array" and "itemSeparator" not in binding:
  st["inputBinding"] = {}
- for k in ("secondaryFiles", "format", "streamable"):
+ for k in ("secondaryFiles", "format", "streamable", "metadata"):
  if k in schema:
  st[k] = schema[k]
  if value_from_expression:

cwltool/command_line_tool.py

@@ -220,6 +220,27 @@ def check_valid_locations(fs_access, ob):
  if ob["class"] == "Directory" and not fs_access.isdir(ob["location"]):
  raise validate.ValidationException("Does not exist or is not a Directory: '%s'" % ob["location"])
 
+def intersect_pair(dict_1, dict_2):
+ # type: (MutableMapping[Text, Any], MutableMapping[Text, Any]) -> MutableMapping[Text, Any]
+ out_dict = dict()
+ for k, v in dict_1.items():
+ if k in dict_2 and dict_2[k] == v:
+ out_dict[k] = v
+ return out_dict
+
+def metadata_intersection(file_list):
+ # type: (List[MutableMapping[Text, Any]]) -> MutableMapping[Text, Any]
+ """
+ Return intersected metadata for a list of files.
+ """
+ if not file_list:
+ return dict()
+ out_metadata = file_list[0].get('metadata', dict())
+ for file_o in file_list:
+ new_metadata = file_o.get('metadata', dict())
+ out_metadata = intersect_pair(out_metadata, new_metadata)
+ return out_metadata
+
 
 OutputPorts = Dict[Text, Union[None, Text, List[Union[Dict[Text, Any], Text]], Dict[Text, Any]]]
 
@@ -759,6 +780,24 @@ def collect_output(self,
  sfitem["class"] = "Directory"
  primary["secondaryFiles"].append(sfitem)
 
+ if "addMetadata" in binding:
+ add_metadata = builder.do_eval(binding["addMetadata"])
+ if isinstance(add_metadata, MutableMapping):
+ metadata_dict = add_metadata
+ else:
+ input_list = []
+ for input_id in aslist(add_metadata):
+ # For union type lists only take files and folders
+ input_list.extend([f for f in aslist(builder.job[input_id])
+ if isinstance(f, MutableMapping)])
+ metadata_dict = metadata_intersection(input_list)
+ for primary in aslist(r):
+ if isinstance(primary, MutableMapping):
+ primary["metadata"] = metadata_dict
+ if "secondaryFiles" in primary:
+ for sf in primary["secondaryFiles"]:
+ sf["metadata"] = metadata_dict
+
  if "format" in schema:
  for primary in aslist(r):
  primary["format"] = builder.do_eval(schema["format"], context=primary)

cwltool/schemas/v1.1.0-dev1/CommandLineTool.yml

@@ -223,6 +223,7 @@ $graph:
  - loadContents
  - outputEval
  - secondaryFiles
+ - addMetadata
  fields:
  - name: glob
  type:
@@ -251,6 +252,23 @@ $graph:
  array of a single element. Additionally, if `loadContents` is `true`,
  the File objects must include up to the first 64 KiB of file contents
  in the `contents` field.
+ - name: addMetadata
+ type:
+ - "null"
+ - string
+ - Expression
+ - string[]
+ doc: |
+ If the value is an expression, expression should return a key-value map.
+ Returned map should be set as the `metadata` property of the output file
+ or files. The same map should be set as the `metadata` property of all
+ secondary files.
+
+ If the value is a string or an array of strings, these must be input
+ parameter references. The `metadata` fields for the referenced inputs are
+ intersected and the resulting key-value map is set as the `metadata`
+ property of the output file or files. The same map should be set as the
+ `metadata` property of all secondary files.
 
 - name: CommandLineBindable
  type: record

cwltool/schemas/v1.1.0-dev1/Process.yml

@@ -279,6 +279,14 @@ $graph:
  If `loadContents` of `inputBinding` or `outputBinding` is true and
  `location` is valid, the implementation must read up to the first 64
  KiB of text from the file and place it in the "contents" field.
+ - name: metadata
+ type: Any?
+ doc: |
+ Metadata should be a key-value map further describing a `File`.
+ Metadata can be propagated from inputs to outputs of a tool using
+ `addMetadata` option in the `CommandOutputBinding`. It is up to the
+ implementation to make use of the `metadata` property of final outputs
+ of the execution.
 
 
 - name: Directory