Add methods to read/write multiple-document YAML files (#1085)

Author: rakillen

core/src/main/java/oracle/weblogic/deploy/yaml/AbstractYamlTranslator.java

@@ -39,55 +39,84 @@ public abstract class AbstractYamlTranslator {
 
  protected abstract PlatformLogger getLogger();
  protected abstract String getClassName();
- public abstract PyDictionary parse() throws YamlException;
+
+ // override to return a list of documents as Python dictionaries from the YAML
+ public abstract PyList parseDocuments(boolean allowMultiple) throws YamlException;
+
+ // override to write a list of documents as Python dictionaries to the YAML
+ public abstract void dumpDocuments(List<?> documents) throws YamlException;
 
  protected AbstractYamlTranslator(String fileName, boolean useOrderedDict) {
  this.fileName = fileName;
  this.useOrderedDict = useOrderedDict;
  }
 
+ /**
+ * This method triggers parsing of the file and conversion into a Python dictionary.
+ * If the YAML is empty, a single empty dictionary is returned.
+ * If the YAML contains multiple documents, a YamlException is thrown
+ *
+ * @return the python dictionary corresponding to the YAML input file
+ * @throws YamlException if an error occurs while reading the input file
+ */
+ public PyDictionary parse() throws YamlException {
+ final String METHOD = "parse";
+ PyDictionary result;
+ PyList dictionaries = parseDocuments(false);
+ if (dictionaries.isEmpty()) {
+ result = getNewDictionary();
+ } else {
+ result = (PyDictionary) dictionaries.get(0);
+ }
+ return result;
+ }
+
  @SuppressWarnings("WeakerAccess")
- protected PyDictionary parseInternal(InputStream inputStream) throws YamlException {
+ protected PyList parseInternal(InputStream inputStream, boolean allowMultiple) throws YamlException {
  final String METHOD = "parseInternal";
 
- PyDictionary result = null;
+ // there are problems using PyList.add(),
+ // so build a java.util.List and construct PyList(javaList).
+ List<PyObject> result = new ArrayList<>();
  if (inputStream != null) {
  Yaml parser = new Yaml(this.getDefaultLoaderOptions());
 
- PyDictionary firstDoc = null;
- int docCount = 0;
  try {
  Iterable<Object> docsIterable = parser.loadAll(inputStream);
-
- Object docToConvert = null;
+ List<Object> documents = new ArrayList<>();
  for (Object document : docsIterable) {
- docCount++;
- if (docCount == 1) {
- docToConvert = document;
- }
+ documents.add(document);
  }
- if (docCount == 1) {
- firstDoc = convertJavaDataStructureToPython(docToConvert);
- } else if (docCount == 0) {
- firstDoc = getNewDictionary();
+
+ // don't continue with conversion if multiple documents check fails
+ if(!allowMultiple && documents.size() > 1) {
+ YamlException pex = new YamlException("WLSDPLY-18101", this.fileName, documents.size());
+ getLogger().throwing(getClassName(), METHOD, pex);
+ throw pex;
  }
+
+ for (Object document : documents) {
+ result.add(convertJavaDataStructureToPython(document));
+ }
+ } catch (YamlException yex) {
+ throw yex;
  } catch (Exception ex) {
  YamlException pex = new YamlException("WLSDPLY-18100", ex, this.fileName, ex.getLocalizedMessage());
  getLogger().throwing(getClassName(), METHOD, pex);
  throw pex;
  }
- if (docCount > 1) {
- YamlException pex = new YamlException("WLSDPLY-18101", this.fileName, docCount);
- getLogger().throwing(getClassName(), METHOD, pex);
- throw pex;
- }
- result = firstDoc;
  }
- return result;
+ return new PyList(result.toArray(new PyObject[0]));
+ }
+
+ public void dump(Map<String, Object> data) throws YamlException {
+ List<Map<String, Object>> list = new ArrayList<>();
+ list.add(data);
+ dumpDocuments(list);
  }
 
  @SuppressWarnings("WeakerAccess")
- protected void dumpInternal(Map<String, Object> data, Writer outputWriter) throws YamlException {
+ protected void dumpInternal(List<?> data, Writer outputWriter) throws YamlException {
  final String METHOD = "dumpInternal";
 
  if (outputWriter != null) {
@@ -96,7 +125,7 @@ protected void dumpInternal(Map<String, Object> data, Writer outputWriter) throw
  Yaml yaml = new Yaml(representer, dumperOptions);
 
  try {
- yaml.dump(replaceNoneInMap(data), outputWriter);
+ yaml.dumpAll(data.iterator(), outputWriter);
  } catch (Exception ex) {
  YamlException pex = new YamlException("WLSDPLY-18107", ex, this.fileName, ex.getLocalizedMessage());
  getLogger().throwing(getClassName(), METHOD, pex);

core/src/main/java/oracle/weblogic/deploy/yaml/YamlStreamTranslator.java

@@ -7,12 +7,12 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Writer;
-import java.util.Map;
+import java.util.List;
 
 import oracle.weblogic.deploy.logging.PlatformLogger;
 import oracle.weblogic.deploy.logging.WLSDeployLogFactory;
 
-import org.python.core.PyDictionary;
+import org.python.core.PyList;
 
 /**
  * An implementation of the YAML parser/translator that reads the YAML input from an input stream.
@@ -63,21 +63,21 @@ public YamlStreamTranslator(String streamFileName, Writer yamlOutputWriter) {
  }
 
  /**
- * This method triggers parsing of the YAML and conversion into the Python dictionary. Note that is closes
- * the input stream when it is finished, making the instance no longer viable.
+ * Read a list of documents as Python dictionaries from the YAML input stream.
+ * Note that the input stream is closed when it is finished, making the instance no longer viable.
  *
- * @return the python dictionary corresponding to the YAML input
+ * @return a list of Python dictionaries corresponding to the YAML input
  * @throws YamlException if an error occurs while reading the input
  */
  @Override
- public PyDictionary parse() throws YamlException {
- final String METHOD = "parse";
+ public PyList parseDocuments(boolean allowMultiple) throws YamlException {
+ final String METHOD = "parseDocuments";
 
  LOGGER.entering(CLASS, METHOD);
- PyDictionary result = null;
+ PyList result = null;
  if (yamlStream != null) {
  try {
- result = parseInternal(yamlStream);
+ result = parseInternal(yamlStream, allowMultiple);
  } finally {
  try {
  yamlStream.close();
@@ -91,8 +91,8 @@ public PyDictionary parse() throws YamlException {
  return result;
  }
 
- public void dump(Map<String, Object> data) throws YamlException {
- final String METHOD = "dump";
+ public void dumpDocuments(List<?> data) throws YamlException {
+ final String METHOD = "dumpDocuments";
 
  LOGGER.entering(CLASS, METHOD);
  if (yamlOutputWriter != null) {

core/src/main/java/oracle/weblogic/deploy/yaml/YamlTranslator.java

@@ -8,13 +8,13 @@
 import java.io.FileInputStream;
 import java.io.FileWriter;
 import java.io.IOException;
-import java.util.Map;
+import java.util.List;
 
 import oracle.weblogic.deploy.logging.PlatformLogger;
 import oracle.weblogic.deploy.logging.WLSDeployLogFactory;
 import oracle.weblogic.deploy.util.FileUtils;
 
-import org.python.core.PyDictionary;
+import org.python.core.PyList;
 
 /**
  * An implementation of the YAML parser/translator that reads the YAML input from an input stream.
@@ -46,20 +46,21 @@ public YamlTranslator(String fileName, boolean useOrderedDict) {
  super(fileName, useOrderedDict);
  this.yamlFile = FileUtils.validateExistingFile(fileName);
  }
+
  /**
- * This method triggers parsing of the file and conversion into the Python dictionary.
+ * Read a list of documents as Python dictionaries from the YAML file.
  *
- * @return the python dictionary corresponding to the YAML input file
- * @throws YamlException if an error occurs while reading the input file
+ * @return a list of documents corresponding to the YAML input
+ * @throws YamlException if an error occurs while reading the input
  */
  @Override
- public PyDictionary parse() throws YamlException {
- final String METHOD = "parse";
+ public PyList parseDocuments(boolean allowMultiple) throws YamlException {
+ final String METHOD = "parseDocuments";
 
  LOGGER.entering(CLASS, METHOD);
- PyDictionary result;
+ PyList result;
  try (FileInputStream fis = new FileInputStream(yamlFile)) {
- result = parseInternal(fis);
+ result = parseInternal(fis, allowMultiple);
  } catch (IOException ioe) {
  YamlException ex = new YamlException("WLSDPLY-18108", ioe, yamlFile.getPath(), ioe.getLocalizedMessage());
  LOGGER.throwing(CLASS, METHOD, ex);
@@ -71,8 +72,8 @@ public PyDictionary parse() throws YamlException {
  return result;
  }
 
- public void dump(Map<String, Object> data) throws YamlException {
- final String METHOD = "dump";
+ public void dumpDocuments(List<?> data) throws YamlException {
+ final String METHOD = "dumpDocuments";
 
  // Don't log the data since it is big and could contain credentials.
  LOGGER.entering(CLASS, METHOD);

core/src/main/python/compare_model.py

@@ -209,7 +209,7 @@ def compare(self):
  print format_message('WLSDPLY-05707')
  print BLANK_LINE
  pty = PythonToYaml(change_model)
- pty._write_dictionary_to_yaml_file(change_model, System.out)
+ pty.write_to_stream(System.out)
 
  return 0
 

core/src/main/python/wlsdeploy/yaml/yaml_translator.py

@@ -14,6 +14,7 @@
 import java.lang.Long as JLong
 import java.lang.String as JString
 import java.util.ArrayList as JArrayList
+from java.io import OutputStreamWriter
 
 import oracle.weblogic.deploy.util.FileUtils as JFileUtils
 import oracle.weblogic.deploy.yaml.YamlStreamTranslator as JYamlStreamTranslator
@@ -63,6 +64,22 @@ def parse(self):
  self._logger.exiting(class_name=self._class_name, method_name=_method_name)
  return result_dict
 
+ def parse_documents(self):
+ """
+ Parse the Yaml content from the file and convert it to a list of Python dictionaries.
+ :return: a list of documents as Python dictionaries
+ :raises: YamlException: if an error occurs while parsing the Yaml or converting it to the list
+ """
+ _method_name = 'parse_documents'
+
+ self._logger.entering(class_name=self._class_name, method_name=_method_name)
+ # throws YamlException with details, nothing we can really add here...
+ result_dict = self._translator.parseDocuments(True)
+
+ # don't log the model on exit, it may contain passwords
+ self._logger.exiting(class_name=self._class_name, method_name=_method_name)
+ return result_dict
+
 
 class YamlStreamToPython(object):
  """
@@ -102,21 +119,24 @@ def parse(self):
 
 class PythonToJava(object):
  """
- A class that converts a Python dictionary and its contents to the Java types expected by snakeyaml.
+ A class that converts a Python dictionary or document list and its contents
+ to the Java types expected by snakeyaml.
  """
  _class_name = 'PythonToJava'
 
- def __init__(self, dictionary):
- self._dictionary = dictionary
+ def __init__(self, collection):
+ self._collection = collection
  self._logger = PlatformLogger('wlsdeploy.yaml')
 
  def convert_to_java(self):
  _method_name = 'convert_to_java'
 
- if self._dictionary is None:
+ if self._collection is None:
  return None
- if isinstance(self._dictionary, dict):
- return self.convert_dict_to_java_map(self._dictionary)
+ if isinstance(self._collection, dict):
+ return self.convert_dict_to_java_map(self._collection)
+ if isinstance(self._collection, list):
+ return self.convert_list_to_java_list(self._collection)
  else:
  yaml_ex = exception_helper.create_yaml_exception('WLSDPLY-18200')
  self._logger.throwing(class_name=self._class_name, method_name=_method_name, error=yaml_ex)
@@ -181,13 +201,13 @@ def convert_scalar_to_java_type(self, py_value):
 
 class PythonToYaml(object):
  """
- A class that converts a Python dictionary into Yaml and writes the output to a file.
+ A class that converts a Python dictionary or document list into Yaml and writes the output to a file.
  """
  _class_name = 'PythonToYaml'
 
- def __init__(self, dictionary):
+ def __init__(self, collection):
  # Fix error handling for None
- self._dictionary = dictionary
+ self._collection = collection
  self._logger = PlatformLogger('wlsdeploy.yaml')
  return
 
@@ -212,7 +232,7 @@ def write_to_yaml_file(self, file_name):
  writer = None
  try:
  writer = JFileWriter(yaml_file)
- self._write_dictionary_to_yaml_file(self._dictionary, writer, file_name)
+ self._write_collection_to_yaml_file(self._collection, writer, file_name)
  except JFileNotFoundException, fnfe:
  yaml_ex = exception_helper.create_yaml_exception('WLSDPLY-18010', file_name,
  fnfe.getLocalizedMessage(), error=fnfe)
@@ -228,20 +248,27 @@ def write_to_yaml_file(self, file_name):
 
  self._logger.exiting(class_name=self._class_name, method_name=_method_name)
 
- def _write_dictionary_to_yaml_file(self, dictionary, writer, file_name='<None>'):
+ def write_to_stream(self, output_stream):
+ self._write_collection_to_yaml_file(self._collection, OutputStreamWriter(output_stream))
+
+ def _write_collection_to_yaml_file(self, collection, writer, file_name='<None>'):
  """
- Do the actual heavy lifting of converting a dictionary and writing it to the file.
- :param dictionary: the Python dictionary to convert
+ Do the actual heavy lifting of converting a dictionary or document list and writing it to the file.
+ :param collection: the Python dictionary or document list to convert
  :param writer: the java.io.Writer for the output file
  :param file_name: the file_name for the output file
  :raises: YamlException: if an error occurs while writing the output
  """
- if dictionary is None:
+ if collection is None:
  return
 
- java_object = PythonToJava(dictionary).convert_to_java()
+ java_object = PythonToJava(collection).convert_to_java()
  yaml_stream_translator = JYamlStreamTranslator(file_name, writer)
- yaml_stream_translator.dump(java_object)
+
+ if isinstance(collection, list):
+ yaml_stream_translator.dumpDocuments(java_object)
+ else:
+ yaml_stream_translator.dump(java_object)
  return
 
  def _close_writer(self, writer):