CU-8693kp0gw: Pin more recent versions for major dependencies; Avoid major bumps where applicable (#392)

* CU-8693kp0gw: Pin more recent versions for major dependencies; Avoid major bumps where applicable

* CU-8693kp0gw: Bump default spacy model version (3.4.0 -> 3.6.0)

* CU-8693kp0gw: Bump fakse test-spacy model version compatibility range

* CU-8693kp0gw: Fix spacy compatibility tests for new fake model

* CU-8693kp0gw: Undo what I thought was a typo fix (moved back from 'movar~virus' to 'movar~viruse')

Author: mart-r

requirements-dev.txt

@@ -1,5 +1,5 @@
 .
-https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.4.0/en_core_web_md-3.4.0-py3-none-any.whl
+https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.6.0/en_core_web_md-3.6.0-py3-none-any.whl
 flake8==4.0.1
 mypy==1.0.0
 mypy-extensions==0.4.3

requirements.txt

@@ -1,2 +1,2 @@
 .
-https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.4.0/en_core_web_md-3.4.0-py3-none-any.whl
+https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.6.0/en_core_web_md-3.6.0-py3-none-any.whl

setup.py

@@ -18,18 +18,18 @@
               'medcat.tokenizers', 'medcat.utils.meta_cat', 'medcat.pipeline', 'medcat.utils.ner',
               'medcat.utils.saving', 'medcat.utils.regression', 'medcat.stats'],
     install_requires=[
-        'numpy>=1.22.0', # first to support 3.11
+        'numpy>=1.22.0,<1.26.0',  # 1.22.0 is first to support python 3.11; post 1.26.0 there's issues with scipy
         'pandas>=1.4.2', # first to support 3.11
-        'gensim>=4.3.0', # first to support 3.11
-        'spacy>=3.1.0',
-        'scipy~=1.9.2', # first to support 3.11
-        'transformers>=4.34.0',
+        'gensim>=4.3.0,<5.0.0',  # 5.3.0 is first to support 3.11; avoid major version bump
+        'spacy>=3.6.0,<4.0.0',  # Some later model packs (e.g HPO) are made with 3.6.0 spacy model; avoid major version bump
+        'scipy~=1.9.2',  # 1.9.2 is first to support 3.11
+        'transformers>=4.34.0,<5.0.0',  # avoid major version bump
         'accelerate>=0.23.0', # required by Trainer class in de-id
-        'torch>=1.13.0', # first to support 3.11
+        'torch>=1.13.0,<3.0.0', # 1.13 is first to support 3.11; 2.1.2 has been compatible, but avoid major 3.0.0 for now
         'tqdm>=4.27',
-        'scikit-learn>=1.1.3', # first to supporrt 3.11
-        'dill>=0.3.4', # allow later versions with later versions of datasets (tested with 0.3.6)
-        'datasets>=2.2.2', # allow later versions, tested with 2.7.1
+        'scikit-learn>=1.1.3,<2.0.0',  # 1.1.3 is first to supporrt 3.11; avoid major version bump
+        'dill>=0.3.6,<1.0.0', # stuff saved in 0.3.6/0.3.7 is not always compatible with 0.3.4/0.3.5; avoid major bump
+        'datasets>=2.2.2,<3.0.0', # avoid major bump
         'jsonpickle>=2.0.0', # allow later versions, tested with 3.0.0
         'psutil>=5.8.0',
         # 0.70.12 uses older version of dill (i.e less than 0.3.5) which is required for datasets

tests/resources/ff_core_fake_dr/meta.json

@@ -4,5 +4,5 @@
     "version":"3.1.0",
     "description":"This is a FAKE model",
     "author":"Fakio Martimus",
-    "spacy_version":">=3.1.0,<3.2.0"
+    "spacy_version":">=3.1.0,<4.0.0"
   }

tests/test_ner.py

@@ -67,8 +67,7 @@ def tearDownClass(cls) -> None:
         cls.pipe.destroy()
 
     def test_aa_cdb_names_output(self):
-        print("Fixing 'movar~viruse' -> 'movar-virus' for newere en_core_web_md")
-        target_result = {'S-229004': {'movar~virus', 'movar', 'movar~viruses'}, 'S-229005': {'cdb'}}
+        target_result = {'S-229004': {'movar~viruse', 'movar', 'movar~viruses'}, 'S-229005': {'cdb'}}
         self.assertEqual(self.cdb.cui2names, target_result)
 
     def test_ab_entities_length(self):

tests/utils/test_spacy_compatibility.py

@@ -149,7 +149,7 @@ def test_reads_info(self):
 
 
 class GetSpacyModelVersionTests(GetSpacyModelInfoTests):
-    expected_spacy_version = ">=3.1.0,<3.2.0"
+    expected_spacy_version = ">=3.1.0,<4.0.0"
 
     @classmethod
     def setUpClass(cls) -> None:
@@ -253,9 +253,9 @@ def test_is_in_range(self):
             self.assertTrue(b)
 
 class ModelPackHasInCompatibleSpacyRangeTests(unittest.TestCase):
-    test_spacy_version = "3.2.0"
+    test_spacy_version = "3.0.0"
 
-    def test_is_in_range(self):
+    def test_is_not_in_range(self):
         with custom_spacy_version(self.test_spacy_version):
             b = medcat_model_pack_has_compatible_spacy_model(FAKE_MODELPACK_MODEL_DIR)
             self.assertFalse(b)