bugfix: read.Conll has to reimplement read_tree_from_lines()

because read.Conllu does not call parse_node_line() anymore (for speed reasons)

Author: martinpopel

udapi/block/read/conll.py

@@ -93,7 +93,69 @@ def parse_node_line(self, line, root, nodes, parents, mwts):
  setattr(node, 'ord', int(fields[n_attribute]))
  elif attribute_name == 'deps':
  setattr(node, 'raw_deps', fields[n_attribute])
- elif attribute_name != '_':
+ elif attribute_name != '_' and fields[n_attribute] != '_':
  setattr(node, attribute_name, fields[n_attribute])
 
  nodes.append(node)
+
+ # Acknowledged code duplication with read.Conllu
+ def read_tree_from_lines(self, lines):
+ root = Root()
+ nodes = [root]
+ parents = [0]
+ mwts = []
+ for line in lines:
+ if line[0] == '#':
+ self.parse_comment_line(line, root)
+ else:
+ self.parse_node_line(line, root, nodes, parents, mwts)
+
+ # If no nodes were read from the filehandle (so only root remained in nodes),
+ # we return None as a sign of failure (end of file or more than one empty line).
+ if len(nodes) == 1:
+ return None
+
+ # Empty sentences are not allowed in CoNLL-U,
+ # but if the users want to save just the sentence string and/or sent_id
+ # they need to create one artificial node and mark it with Empty=Yes.
+ # In that case, we will delete this node, so the tree will have just the (technical) root.
+ # See also udapi.block.write.Conllu, which is compatible with this trick.
+ if len(nodes) == 2 and str(nodes[1].misc) == 'Empty=Yes':
+ nodes.pop()
+ root._children = []
+ root._descendants = []
+
+ # Set dependency parents (now, all nodes of the tree are created).
+ for node_ord, node in enumerate(nodes[1:], 1):
+ try:
+ parent = nodes[parents[node_ord]]
+ except IndexError:
+ raise ValueError("Node %s HEAD is out of range (%d)" % (node, parents[node_ord]))
+ if node is parent:
+ if self.fix_cycles:
+ logging.warning("Ignoring a cycle (attaching to the root instead):\n%s", node)
+ node._parent = root
+ root._children.append(node)
+ else:
+ raise ValueError(f"Detected a cycle: {node} attached to itself")
+ elif node.children:
+ climbing = parent._parent
+ while climbing:
+ if climbing is node:
+ if self.fix_cycles:
+ logging.warning("Ignoring a cycle (attaching to the root instead):\n%s", parent)
+ parent = root
+ break
+ else:
+ raise ValueError(f"Detected a cycle: {node}")
+ climbing = climbing._parent
+ node._parent = parent
+ parent._children.append(node)
+
+ # Create multi-word tokens.
+ for fields in mwts:
+ range_start, range_end = fields[0].split('-')
+ words = nodes[int(range_start):int(range_end) + 1]
+ root.create_multiword_token(words, form=fields[1], misc=fields[-1])
+
+ return root

udapi/core/mwt.py

@@ -9,7 +9,7 @@ class MWT(object):
  def __init__(self, words=None, form=None, misc=None, root=None):
  self.words = words if words is not None else []
  self.form = form
- self._misc = DualDict(misc) if misc else None
+ self._misc = DualDict(misc) if misc and misc != '_' else None
  self.root = root
  for word in self.words:
  word._mwt = self # pylint: disable=W0212
@@ -49,7 +49,7 @@ def address(self):
 
 # TODO: node.remove() should check if the node is not part of any MWT
 # TODO: Document that editing words by mwt.words.append(node), del or remove(node) is not supported
-# TODO: Make mwt._words privat and provide a setter
+# TODO: Make mwt._words private and provide a setter
 # TODO: What to do when mwt.words = []? (It is allowed after mwt=MWT().)
 # TODO: words.setter and node.shift* should check if the MWT does not contain gaps
 # and is still multi-word