add fuzzy algorithm based on simstring algorithm called from pysimstring library

Author: sebastien cossin

docs/source/api_doc.rst

@@ -238,7 +238,21 @@ ESpellWiseAlgo
  :undoc-members:
  :show-inheritance:
 
+SimString
+^^^^^^^^^
+SimStringWrapper
+""""""""""""""""
+.. autoclass:: iamsystem.fuzzy.SimStringWrapper
+ :members:
+ :undoc-members:
+ :show-inheritance:
 
+ESimStringMeasure
+"""""""""""""""""
+.. autoclass:: iamsystem.fuzzy.ESimStringMeasure
+ :members:
+ :undoc-members:
+ :show-inheritance:
 
 Brat
 ----

docs/source/fuzzy.rst

@@ -139,9 +139,15 @@ if you use a custom tokenizer (i.e. from an external library like spaCy) you can
 String Distance
 ^^^^^^^^^^^^^^^
 .. _spellwise: https://github.com/chinnichaitanya/spellwise
+.. _pysimstring: https://github.com/percevalw/pysimstring
 
-This package utilizes the `spellwise`_ python library to access string distance algorithms.
-In the example below, iamsystem is configured with two spelling algorithms:
+
+This package utilizes the `spellwise`_ and `pysimstring`_ libraries to access string distance algorithms.
+
+Spellwise
+"""""""""
+
+In the example below, iamsystem is configured with two spellwise algorithms:
 Levenshtein distance which measures the number of edits needed to transform one word into another,
 and Soundex which is a phonetic algorithm.
 
@@ -181,6 +187,40 @@ When the number of keywords is large, these algorithms can be slow.
 Since their output doesn't depend on the context,
 I recommend using the :ref:`fuzzy:CacheFuzzyAlgos` class to store them.
 
+simstring
+"""""""""
+.. _simstring: http://chokkan.org/software/simstring/
+
+The `pysimstring`_ library provides an API to the fast `simstring`_ algorithm implemented in C++.
+
+In the example below, all the unigrams of the keywords are indexed by simstring.
+Then, for each token in the document, simstring is called to return the closed matched.
+
+.. code-block:: python
+
+ from iamsystem.fuzzy.simstring import (
+ SimStringWrapper,
+ ESimStringMeasure,
+ )
+ from iamsystem import Term, Matcher
+
+ term1 = Term(label="acute respiratory distress", code="J80")
+ matcher = Matcher()
+ matcher.add_keywords(keywords=[term1])
+ fuzzy_ss = SimStringWrapper(
+ words=matcher.get_keywords_unigrams(),
+ measure=ESimStringMeasure.COSINE,
+ threshold=0.7,
+ )
+ matcher.add_fuzzy_algo(fuzzy_algo=fuzzy_ss)
+ annots = matcher.annot_text(text="acute respiratori disstress")
+ for annot in annots:
+ print(annot)
+ # acute respiratori disstress	0 27	acute respiratory distress (J80)
+
+Using the cosine similarity and a threshold of 0.7,
+the tokens *respiratori* matched to *respiratory* and *disstress* matched to *distress*.
+
 CacheFuzzyAlgos
 ^^^^^^^^^^^^^^^
 

pyproject.toml

@@ -20,7 +20,7 @@ keywords = ["NLP", "semantic annotation", "entity linking"]
 dependencies = ['unidecode>=1.1.1', 'typing-extensions~=4.4.0', 'spellwise>=0.8.0']
 
 [project.optional-dependencies]
-tests = ['spellwise>=0.8.0', 'nltk>=3.8.0', 'spacy>=3.2.0']
+tests = ['spellwise>=0.8.0', 'nltk>=3.8.0', 'spacy>=3.2.0', 'pysimstring==1.2.1']
 doc = ["sphinx"]
 
 [project.urls]

src/iamsystem/fuzzy/simstring.py

@@ -0,0 +1,94 @@
+""" pysimstring library wrapper."""
+import os
+import tempfile
+
+from enum import Enum
+from typing import Iterable
+
+from pysimstring import simstring
+
+from iamsystem.fuzzy.api import NormLabelAlgo
+from iamsystem.fuzzy.api import SynType
+
+
+class ESimStringMeasure(Enum):
+ """Enumerated list of simstring measures."""
+
+ EXACT = "exact"
+ DICE = "dice"
+ COSINE = "cosine"
+ JACCARD = "jaccard"
+ OVERLAP = "overlap"
+
+
+class SimStringWrapper(NormLabelAlgo):
+ """SimString algorithm interface."""
+
+ def __init__(
+ self,
+ words=Iterable[str],
+ name: str = "simstring",
+ measure=ESimStringMeasure.JACCARD,
+ threshold=0.5,
+ ):
+ """Create a fuzzy algorithm that calls simstring.
+
+ :param words: the words to index in the simstring database.
+ An easy way to provide these words is to call
+ :py:meth:`~iamsystem.Matcher.get_keywords_unigrams` method after
+ you added your keywords to the matcher instance.
+ :param name: a name given to this algorithm. Default "simstring".
+ :param measure: a similarity measure selected from
+ :class:`~iamsystem.fuzzy.simstring.ESimStringMeasure`.
+ Default JACCARD.
+ :param threshold: similarity measure threshold.
+ """
+ super().__init__(name=name)
+ self.path = tempfile.mkdtemp()
+ os.makedirs(self.path, exist_ok=True)
+ abs_path = os.path.join(self.path, "terms.simstring")
+ with SimstringWriter(abs_path=abs_path) as ss_db:
+ for word in words:
+ ss_db.insert(word)
+ self.ss_reader = simstring.reader(abs_path)
+ self.ss_reader.measure = getattr(simstring, measure.value)
+ self.ss_reader.threshold = threshold
+
+ def get_syns_of_word(self, word: str) -> Iterable[SynType]:
+ """Retrieve simstring similar words."""
+ ss_words = self.ss_reader.retrieve(word)
+ return [self.word_to_syn(word=word) for word in ss_words]
+
+ def __del__(self):
+ """close the file connection to simstring db."""
+ # The safer approach is to open the file for every 'get_syns_of_word'
+ # call. However, it takes more time. It seems to be ok to close
+ # the file here.
+ # https://stackoverflow.com/questions/44142836/open-file-inside-class
+ self.ss_reader.close()
+
+
+class SimstringWriter:
+ """Utility class to create a simstring database.
+ a plagiarism of https://github.com/percevalw/pysimstring/blob/master/tests/test_simstring.py # noqa
+ """
+
+ def __init__(self, abs_path: str):
+ """A context class to write a simstring database
+
+ :param abs_path: absolute path to the file.
+ """
+ self.abs_path = abs_path
+
+ def __enter__(self):
+ """Open the file"""
+ self.db = simstring.writer(self.abs_path, 3, False, True)
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ """Close the file"""
+ self.db.close()
+
+ def insert(self, term):
+ """insert a term in simstring file."""
+ self.db.insert(term)

tests/test_doc.py

@@ -575,6 +575,28 @@ def test_spellwise(self):
  # acute resiratory distresssss	0 28	acute respiratory distress (J80) acute(exact,LEVENSHTEIN,SOUNDEX);resiratory(LEVENSHTEIN);distresssss(SOUNDEX) # noqa
  self.assertEqual(1, len(annots))
 
+ def test_simstring(self):
+ """Simstring example."""
+ from iamsystem import Matcher
+ from iamsystem import Term
+ from iamsystem.fuzzy.simstring import ESimStringMeasure
+ from iamsystem.fuzzy.simstring import SimStringWrapper
+
+ term1 = Term(label="acute respiratory distress", code="J80")
+ matcher = Matcher()
+ matcher.add_keywords(keywords=[term1])
+ fuzzy_ss = SimStringWrapper(
+ words=matcher.get_keywords_unigrams(),
+ measure=ESimStringMeasure.COSINE,
+ threshold=0.7,
+ )
+ matcher.add_fuzzy_algo(fuzzy_algo=fuzzy_ss)
+ annots = matcher.annot_text(text="acute respiratori disstress")
+ for annot in annots:
+ print(annot)
+ # acute respiratori disstress	0 27	acute respiratory distress (J80)
+ self.assertEqual(1, len(annots))
+
  def test_cache_fuzzy_algos(self):
  """Cache example."""
  from iamsystem import Abbreviations

tests/test_simstring.py

@@ -0,0 +1,112 @@
+import unittest
+
+from iamsystem import CacheFuzzyAlgos
+from iamsystem import Matcher
+from iamsystem.fuzzy.simstring import ESimStringMeasure
+from iamsystem.fuzzy.simstring import SimStringWrapper
+
+
+class MatcherTest(unittest.TestCase):
+ def setUp(self) -> None:
+ self.matcher = Matcher()
+ self.matcher.add_labels(labels=["paracetamol", "les"])
+
+ def test_threshold_1(self):
+ """Test threshold=1 is exact match"""
+ fuzzy_ss = SimStringWrapper(words=["paracetamol"], threshold=1)
+ syns = list(fuzzy_ss.get_syns_of_word("paracetomol"))
+ self.assertEqual(0, len(syns))
+ syns = list(fuzzy_ss.get_syns_of_word("paracetamol"))
+ self.assertEqual(1, len(syns))
+
+ def test_threshold_0_5(self):
+ """Test synonyms returned depend on threshold."""
+ fuzzy_ss = SimStringWrapper(words=["paracetamol"])
+ syns = list(fuzzy_ss.get_syns_of_word("paracetomol"))
+ self.assertEqual(1, len(syns))
+ syns = list(fuzzy_ss.get_syns_of_word("para"))
+ self.assertEqual(0, len(syns))
+
+ def test_threshold_0_2(self):
+ """Test synonyms returned depend on threshold."""
+ fuzzy_ss = SimStringWrapper(words=["paracetamol"], threshold=0.2)
+ syns = list(fuzzy_ss.get_syns_of_word("paracetomol"))
+ self.assertEqual(1, len(syns))
+ syns = list(fuzzy_ss.get_syns_of_word("para"))
+ self.assertEqual(1, len(syns))
+
+ def test_measure_exact(self):
+ """Test synonyms returned depend on threshold."""
+ fuzzy_ss = SimStringWrapper(
+ words=["paracetamol"], measure=ESimStringMeasure.EXACT
+ )
+ syns = list(fuzzy_ss.get_syns_of_word("paracetomol"))
+ self.assertEqual(0, len(syns))
+ syns = list(fuzzy_ss.get_syns_of_word("paracetamol"))
+ self.assertEqual(1, len(syns))
+
+ def test_other_measures(self):
+ """Test other similarity measure ; check it returns a synonym."""
+ for measure in ESimStringMeasure:
+ if measure.value == "exact":
+ continue
+ fuzzy_ss = SimStringWrapper(words=["paracetamol"], measure=measure)
+ syns = list(fuzzy_ss.get_syns_of_word("paracetomol"))
+ self.assertEqual(1, len(syns))
+
+ def test_matcher(self):
+ """Test detection with a matcher"""
+ fuzzy_ss = SimStringWrapper(words=self.matcher.get_keywords_unigrams())
+ self.matcher.add_fuzzy_algo(fuzzy_algo=fuzzy_ss)
+ annots = self.matcher.annot_text(text="le paractamol")
+ self.assertEqual(1, len(annots))
+
+ def test_cache_fuzzy_algos(self):
+ """Test it can work with CacheFuzzyAlgos."""
+ fuzzy_ss = SimStringWrapper(words=self.matcher.get_keywords_unigrams())
+ cache = CacheFuzzyAlgos()
+ cache.add_algo(algo=fuzzy_ss)
+ self.matcher.add_fuzzy_algo(fuzzy_algo=cache)
+ annots = self.matcher.annot_text(text="le paractamol")
+ self.assertEqual(1, len(annots))
+
+ def test_combine_multiple_algos(self):
+ """Test we can add multiple simstring algorithms."""
+ fuzzy_dice = SimStringWrapper(
+ words=self.matcher.get_keywords_unigrams(),
+ name="ss_dice",
+ measure=ESimStringMeasure.DICE,
+ )
+ fuzzy_jaccard = SimStringWrapper(
+ words=self.matcher.get_keywords_unigrams(),
+ name="ss_jaccard",
+ measure=ESimStringMeasure.JACCARD,
+ )
+ self.matcher.add_fuzzy_algo(fuzzy_algo=fuzzy_dice)
+ self.matcher.add_fuzzy_algo(fuzzy_algo=fuzzy_jaccard)
+ annots = self.matcher.annot_text(text="le paractamol")
+ self.assertEqual(1, len(annots))
+ annot = annots[0]
+ algos_token_0 = annot.algos[0]
+ self.assertEqual(["ss_dice", "ss_jaccard"], algos_token_0)
+
+ def test_combine_multiple_algos_2(self):
+ """Test the two simstring databases are independent to allow
+ the user to customize different files."""
+ fuzzy_dice = SimStringWrapper(
+ words=self.matcher.get_keywords_unigrams(),
+ name="ss_dice",
+ measure=ESimStringMeasure.DICE,
+ )
+ fuzzy_jaccard = SimStringWrapper(
+ words=["NothingInterestingHere"],
+ name="ss_jaccard",
+ measure=ESimStringMeasure.JACCARD,
+ )
+ self.matcher.add_fuzzy_algo(fuzzy_algo=fuzzy_dice)
+ self.matcher.add_fuzzy_algo(fuzzy_algo=fuzzy_jaccard)
+ annots = self.matcher.annot_text(text="le paractamol")
+ self.assertEqual(1, len(annots))
+ annot = annots[0]
+ algos_token_0 = annot.algos[0]
+ self.assertEqual(["ss_dice"], algos_token_0)