Remove add_labels function, refactor, update README, add ChangeLog

Author: scossin

CHANGELOG.md

@@ -0,0 +1,28 @@
+# ChangeLog
+
+## Version 0.2.0 (beta)
+
+### Breaking changes
+
+- IMatcher interface: the window size parameter *w* was removed from the *annot_text* function.
+In the Matcher class, this parameter becomes an attribute.
+- Matcher: remove *add_labels* function, *add_keywords* function supports an Iterable of string labels.
+- SpellWiseWrapper init: 'spellwise_algo' argument renamed 'measure' to be consistent with string distance algorithm.
+- SpellWiseWrapper *add_words_to_ignore* is deprecated, moved to the init function.
+- Fuzzyregex init: 'algo_name' argument renamed 'name'.
+
+
+### Enhancement
+
+- Added support for pysimstring library (string distance fuzzy algorithm).
+- Created StringDistance, parent class of pysimstring and spellwise wrapper.
+- Created IWords2ignore for StringDistance algorithms to ignore common words.
+This speeds up these algorithms and reduces the false positive rate.
+- Added a Matcher.build() function that greatly simplifies the construction of the Matcher.
+The documentation has been updated accordingly.
+- Added an *IBaseMatcher* interface to be the main interface of any IAMsystem matcher (currently only one)
+which should not be changed in the future.
+
+## Version 0.1.1 (alpha)
+
+### Initial release

README.md

@@ -32,7 +32,7 @@ matcher = Matcher.build(
  keywords=["North America", "South America"],
  stopwords=["and"],
  abbreviations=[("amer", "America")],
- spellwise=[dict(algo=ESpellWiseAlgo.LEVENSHTEIN, max_distance=1)],
+ spellwise=[dict(measure=ESpellWiseAlgo.LEVENSHTEIN, max_distance=1)],
  w=2,
 )
 annots = matcher.annot_text(text="Northh and south Amer.")
@@ -46,13 +46,16 @@ for annot in annots:
 ## Algorithm
 The algorithm was developed in the context of a [PhD thesis](https://theses.hal.science/tel-03857962/).
 It proposes a solution to quickly annotate documents using a large dictionary (> 300K keywords) and fuzzy matching algorithms.
-No string distance algorithm is implemented in this package, it imports and leverages external libraries like [spellwise](https://github.com/chinnichaitanya/spellwise)
-and [nltk](https://github.com/nltk/nltk).
-Its algorithmic complexity is O(n(log(m))) with n the number of tokens in a document and m the size of the dictionary.
+No string distance algorithm is implemented in this package, it imports and leverages external libraries like [spellwise](https://github.com/chinnichaitanya/spellwise),
+[pysimstring](https://github.com/percevalw/pysimstring) and [nltk](https://github.com/nltk/nltk).
+Its algorithmic complexity is *O(n(log(m)))* with n the number of tokens in a document and m the size of the dictionary.
 The formalization of the algorithm is available in this [paper](https://ceur-ws.org/Vol-3202/livingner-paper11.pdf).
 
-The algorithm was initially developed in Java (https://github.com/scossin/IAMsystem) and
-has participated in several semantic annotation competitions in the medical domain where it has obtained very satisfactory results.
+The algorithm was initially developed in Java (https://github.com/scossin/IAMsystem).
+It has participated in several semantic annotation competitions in the medical field where it has obtained satisfactory results,
+for example by obtaining the best results in the [Codiesp shared task]([https://temu.bsc.es/codiesp/index.php/2019/09/19/awards/).
+A dictionary-based model can achieve close performance to a transformer-based model when the task is simple or when the training set is small.
+Its main advantage is its speed, which allows a baseline to be generated quickly.
 
 ### Citation
 ```
@@ -69,8 +72,3 @@ has participated in several semantic annotation competitions in the medical doma
 keywords = {Computer Science - Computation and Language},
 }
 ```
-
-## Changelog
-
-**0.1.1**
-* First release

src/iamsystem/__init__.py

@@ -1,5 +1,7 @@
 __all__ = [
  "Matcher",
+ "IMatcher",
+ "IBaseMatcher",
  "Annotation",
  "rm_nested_annots",
  "IStopwords",
@@ -71,6 +73,8 @@
 from iamsystem.matcher.annotation import Annotation
 from iamsystem.matcher.annotation import replace_annots
 from iamsystem.matcher.annotation import rm_nested_annots
+from iamsystem.matcher.api import IBaseMatcher
+from iamsystem.matcher.api import IMatcher
 from iamsystem.matcher.matcher import Matcher
 from iamsystem.stopwords.api import IStopwords
 from iamsystem.stopwords.negative import NegativeStopwords

src/iamsystem/fuzzy/spellwise.py

@@ -61,15 +61,15 @@ class SpellWiseWrapper(StringDistance):
 
  def __init__(
  self,
- algo: ESpellWiseAlgo,
+ measure: ESpellWiseAlgo,
  max_distance: int,
  min_nb_char=5,
  words2ignore: Optional[IWords2ignore] = None,
  name: str = None,
  ):
  """Create an instance to take advantage of a spellwise algorithm.
 
- :param algo: A value from :class:`~iamsystem.SpellWiseAlgo`
+ :param measure: A value from :class:`~iamsystem.SpellWiseAlgo`
  enumerated list.
  :param max_distance: maximum edit distance
  (see spellwise documentation).
@@ -81,11 +81,11 @@ def __init__(
  Default: spellwise algorithm's name.
  """
  if name is None:
- name = algo.name
+ name = measure.name
  super().__init__(
  name=name, min_nb_char=min_nb_char, words2ignore=words2ignore
  )
- self._suggester: ISpellWiseAlgo = algo.value()
+ self._suggester: ISpellWiseAlgo = measure.value()
  self._max_distance = max_distance
 
  @property

src/iamsystem/matcher/abstract.py

@@ -191,15 +191,22 @@ def tokenize(self, text: str) -> Sequence[TokenT]:
  """
  return self._tokenizer.tokenize(text=text)
 
- def add_labels(self, labels: Iterable[str]) -> None:
- """Utility function to call 'add_keywords' by providing a list of
- labels, :class:`~iamsystem.IKeyword` instances are created and added.
+ def add_keywords(self, keywords: Iterable[Union[str, IKeyword]]) -> None:
+ """Utility function to add multiple keywords.
 
- :param labels: the labels (keywords) to be searched in the document.
+ :param keywords: an iterable of string (labels) or
+ :class:`~iamsystem.IKeyword` to search in a document.
  :return: None.
  """
- keywords = [Keyword(label=label) for label in labels]
- self.add_keywords(keywords=keywords)
+ for kw in keywords:
+ if isinstance(kw, str):
+ kw = Keyword(label=kw)
+ if not isinstance(kw, IKeyword):
+ raise TypeError(
+ f"{kw.__class__} is neither a string "
+ f"or a class that implements the IKeyword interface."
+ )
+ self.add_keyword(keyword=kw)
 
  def add_keyword(self, keyword: IKeyword) -> None:
  """Add a keyword to find in a document.
@@ -214,15 +221,6 @@ def add_keyword(self, keyword: IKeyword) -> None:
  stopwords=self,
  )
 
- def add_keywords(self, keywords: Iterable[IKeyword]) -> None:
- """Utility function to add multiple keywords.
-
- :param keywords: :class:`~iamsystem.IKeyword` to search in a document.
- :return: None.
- """
- for keyword in keywords:
- self.add_keyword(keyword=keyword)
-
  @property
  def keywords(self) -> Collection[IKeyword]:
  """Return the keywords added."""
@@ -391,16 +389,7 @@ def build(
  matcher.remove_nested_annots = remove_nested_annots
 
  # Add the keywords
- for kw in keywords:
- if isinstance(kw, str):
- matcher.add_labels(labels=[kw])
- elif isinstance(kw, IKeyword):
- matcher.add_keyword(keyword=kw)
- else:
- raise ValueError(
- f"{kw.__class__} is neither a string "
- f"or a class that implements the IKeyword interface."
- )
+ matcher.add_keywords(keywords=keywords)
 
  # add negative stopwords after stopwords and keywords are added
  # since this class needs keywords'unigrams without stopwords.

src/iamsystem/matcher/matcher.py

@@ -17,7 +17,7 @@ def test_rm_nested_terms_right_overlapping(self):
  """Since 'prostate cancer' overlaps 'cancer', 'cancer' is a nested
  annotation to remove. 'cancer' is right-most token."""
  matcher = Matcher()
- matcher.add_labels(labels=["prostate cancer", "cancer"])
+ matcher.add_keywords(keywords=["prostate cancer", "cancer"])
  matcher.remove_nested_annots = False
  annots = matcher.annot_text(text="prostate cancer")
  self.assertEqual(2, len(annots))
@@ -51,7 +51,9 @@ def test_rm_nested_terms_middle(self):
  Check it work with a middle term.
  """
  matcher = Matcher()
- matcher.add_labels(labels=["prostate cancer undocumented", "cancer"])
+ matcher.add_keywords(
+ keywords=["prostate cancer undocumented", "cancer"]
+ )
  matcher.remove_nested_annots = False
  annots = matcher.annot_text(text="prostate cancer undocumented")
  self.assertEqual(2, len(annots))

tests/test_annotation.py

@@ -22,7 +22,9 @@ def test_readme_example(self):
  keywords=["North America", "South America"],
  stopwords=["and"],
  abbreviations=[("amer", "America")],
- spellwise=[dict(algo=ESpellWiseAlgo.LEVENSHTEIN, max_distance=1)],
+ spellwise=[
+ dict(measure=ESpellWiseAlgo.LEVENSHTEIN, max_distance=1)
+ ],
  w=2,
  )
  annots = matcher.annot_text(text="Northh and south Amer.")
@@ -609,11 +611,11 @@ def test_spellwise(self):
  keywords=[term1],
  spellwise=[
  dict(
- algo=ESpellWiseAlgo.LEVENSHTEIN,
+ measure=ESpellWiseAlgo.LEVENSHTEIN,
  max_distance=1,
  min_nb_char=5,
  ),
- dict(algo=ESpellWiseAlgo.SOUNDEX, max_distance=1),
+ dict(measure=ESpellWiseAlgo.SOUNDEX, max_distance=1),
  ],
  )
  annots = matcher.annot_text(text="acute resiratory distresssss")
@@ -633,7 +635,7 @@ def test_string_distance_ignored_w(self):
  keywords=["poids"],
  spellwise=[
  dict(
- algo=ESpellWiseAlgo.LEVENSHTEIN,
+ measure=ESpellWiseAlgo.LEVENSHTEIN,
  max_distance=1,
  min_nb_char=4,
  )
@@ -646,7 +648,7 @@ def test_string_distance_ignored_w(self):
  keywords=["poids"],
  spellwise=[
  dict(
- algo=ESpellWiseAlgo.LEVENSHTEIN,
+ measure=ESpellWiseAlgo.LEVENSHTEIN,
  max_distance=1,
  min_nb_char=4,
  )
@@ -695,7 +697,7 @@ def test_cache_fuzzy_algos(self):
  abbs = Abbreviations(name="abbs")
  abbs.add(short_form="a", long_form="acute", tokenizer=matcher)
  test = dict(
- algo=ESpellWiseAlgo.LEVENSHTEIN, max_distance=1, min_nb_char=5
+ measure=ESpellWiseAlgo.LEVENSHTEIN, max_distance=1, min_nb_char=5
  )
  levenshtein = SpellWiseWrapper(**test)
  soundex = SpellWiseWrapper(ESpellWiseAlgo.SOUNDEX, max_distance=1)

tests/test_doc.py

@@ -68,11 +68,11 @@ def test_detect_exact_match(self):
 
  def test_add_labels(self):
  """This function add keywords that can be detected."""
- detector = Matcher()
+ matcher = Matcher()
  words = ["acute respiratory distress syndrome", "diarrrhea"]
- detector.add_labels(labels=words)
+ matcher.add_keywords(keywords=words)
  text = "Pt c/o acute respiratory distress syndrome and diarrrhea"
- annots = detector.annot_text(text=text)
+ annots = matcher.annot_text(text=text)
  self.assertEqual(2, len(annots))
 
  def test_keywords_attribute(self):
@@ -153,7 +153,7 @@ def test_unordered_words_seq(self):
  matcher = Matcher(tokenizer=tokenizer)
  tokens = matcher.tokenize(text)
  self.assertEqual("d", tokens[0].norm_label)
- matcher.add_labels(labels=["insuffisance ventriculaire gauche"])
+ matcher.add_keywords(keywords=["insuffisance ventriculaire gauche"])
  matcher.w = 10
  annots = matcher.annot_text(text=text)
  self.assertEqual(1, len(annots))
@@ -323,7 +323,9 @@ def test_spellwise(self):
 
  matcher = Matcher.build(
  keywords=get_termino_ivg(),
- spellwise=[dict(algo=ESpellWiseAlgo.LEVENSHTEIN, max_distance=1)],
+ spellwise=[
+ dict(measure=ESpellWiseAlgo.LEVENSHTEIN, max_distance=1)
+ ],
  )
  annots = matcher.annot_text(text=text)
  self.assertEqual(1, len(annots))
@@ -333,7 +335,9 @@ def test_spellwise_param_order(self):
  text = "insuffisance cardiaqu gauche"
  matcher = Matcher.build(
  keywords=get_termino_ivg(),
- spellwise=[dict(max_distance=1, algo=ESpellWiseAlgo.LEVENSHTEIN)],
+ spellwise=[
+ dict(max_distance=1, measure=ESpellWiseAlgo.LEVENSHTEIN)
+ ],
  )
  annots = matcher.annot_text(text=text)
  self.assertEqual(1, len(annots))
@@ -344,7 +348,7 @@ def test_spellwise_wrong_param(self):
  Matcher.build(
  keywords=get_termino_ivg(),
  spellwise=[
- dict(max_distance=1, measure=ESpellWiseAlgo.LEVENSHTEIN)
+ dict(max_distance=1, algo=ESpellWiseAlgo.LEVENSHTEIN)
  ],
  )
 
@@ -359,7 +363,7 @@ def test_spellwise_params_overrides(self):
  spellwise=[
  dict(
  max_distance=1,
- algo=ESpellWiseAlgo.LEVENSHTEIN,
+ measure=ESpellWiseAlgo.LEVENSHTEIN,
  words2ignore=words2ignore,
  )
  ],
@@ -374,7 +378,7 @@ def test_spellwise_params_overrides(self):
  spellwise=[
  dict(
  max_distance=1,
- algo=ESpellWiseAlgo.LEVENSHTEIN,
+ measure=ESpellWiseAlgo.LEVENSHTEIN,
  # words2ignore=words2ignore
  )
  ],

tests/test_matcher.py

@@ -9,7 +9,7 @@
 class MatcherTest(unittest.TestCase):
  def setUp(self) -> None:
  self.matcher = Matcher()
- self.matcher.add_labels(labels=["paracetamol", "les"])
+ self.matcher.add_keywords(keywords=["paracetamol", "les"])
 
  def test_threshold_1(self):
  """Test threshold=1 is exact match"""