Merge pull request #3624 from flairNLP/fix_opus_corpus

fix opus corpus and fix ruff errors

Author: alanakbik

flair/__init__.py

@@ -85,16 +85,16 @@ def set_seed(seed: int):
 
 
 __all__ = [
+    "__version__",
     "cache_root",
+    "data",
+    "datasets",
     "device",
-    "__version__",
     "logger",
-    "set_seed",
-    "data",
     "models",
     "nn",
+    "set_proxies",
+    "set_seed",
     "trainers",
     "visual",
-    "datasets",
-    "set_proxies",
 ]

flair/class_utils.py

@@ -2,8 +2,7 @@
 import inspect
 from collections.abc import Iterable
 from types import ModuleType
-from typing import Any, Iterable, List, Optional, Protocol, Type, TypeVar, Union, overload
-
+from typing import Any, Optional, Protocol, TypeVar, Union, overload
 
 T = TypeVar("T")
 
@@ -12,7 +11,7 @@ class StringLike(Protocol):
     def __str__(self) -> str: ...
 
 
-def get_non_abstract_subclasses(cls: Type[T]) -> Iterable[Type[T]]:
+def get_non_abstract_subclasses(cls: type[T]) -> Iterable[type[T]]:
     for subclass in cls.__subclasses__():
         yield from get_non_abstract_subclasses(subclass)
         if inspect.isabstract(subclass):

flair/data.py

@@ -1389,11 +1389,12 @@ def __init__(
         sample_missing_splits: Union[bool, str] = True,
         random_seed: Optional[int] = None,
     ) -> None:
-        """Constructor method to initialize a :class:`Corpus`. You can define the train, dev and test split
+        """Initialize a Corpus.
+
+        You can define the train, dev and test split
         by passing the corresponding Dataset object to the constructor. At least one split should be defined.
         If the option `sample_missing_splits` is set to True, missing splits will be randomly sampled from the
         train split.
-
         In most cases, you will not use the constructor yourself. Rather, you will create a corpus using one of our
         helper methods that read common NLP filetypes. For instance, you can use
         :class:`flair.datasets.sequence_labeling.ColumnCorpus` to read CoNLL-formatted files directly into
@@ -1679,11 +1680,7 @@ def _count_token_labels(sentences: Iterable[Sentence], label_type: str) -> defau
         return label_count
 
     def __str__(self) -> str:
-        return "Corpus: %d train + %d dev + %d test sentences" % (
-            _len_dataset(self.train) if self.train else 0,
-            _len_dataset(self.dev) if self.dev else 0,
-            _len_dataset(self.test) if self.test else 0,
-        )
+        return f"Corpus: {_len_dataset(self.train) if self.train else 0} train + {_len_dataset(self.dev) if self.dev else 0} dev + {_len_dataset(self.test) if self.test else 0} test sentences"
 
     def make_label_dictionary(
         self, label_type: str, min_count: int = -1, add_unk: bool = False, add_dev_test: bool = False

flair/datasets/__init__.py

@@ -324,19 +324,11 @@
 )
 
 __all__ = [
-    "DataLoader",
-    "OcrJsonDataset",
-    "SROIE",
-    "FlairDatapointDataset",
-    "SentenceDataset",
-    "MongoDataset",
-    "StringDataset",
-    "EntityLinkingDictionary",
     "AGNEWS",
+    "AMAZON_REVIEWS",
     "ANAT_EM",
     "AZDZ",
     "BC2GM",
-    "BIO_INFER",
     "BIOBERT_CHEMICAL_BC4CHEMD",
     "BIOBERT_CHEMICAL_BC5CDR",
     "BIOBERT_DISEASE_BC5CDR",
@@ -347,19 +339,41 @@
     "BIOBERT_SPECIES_S800",
     "BIONLP2013_CG",
     "BIONLP2013_PC",
+    "BIOSCOPE",
     "BIOSEMANTICS",
+    "BIO_INFER",
     "CDR",
     "CELL_FINDER",
     "CEMP",
     "CHEMDNER",
+    "CLEANCONLL",
     "CLL",
+    "COMMUNICATIVE_FUNCTIONS",
+    "CONLL_03",
+    "CONLL_03_DUTCH",
+    "CONLL_03_GERMAN",
+    "CONLL_03_SPANISH",
+    "CONLL_2000",
     "CRAFT",
     "CRAFT_V4",
+    "CTD_CHEMICALS_DICTIONARY",
+    "CTD_DISEASES_DICTIONARY",
     "DECA",
+    "FEWNERD",
     "FSU",
     "GELLUS",
+    "GERMEVAL_2018_OFFENSIVE_LANGUAGE",
+    "GLUE_COLA",
+    "GLUE_MNLI",
+    "GLUE_MRPC",
+    "GLUE_QNLI",
+    "GLUE_QQP",
+    "GLUE_RTE",
+    "GLUE_SST2",
+    "GLUE_STSB",
+    "GLUE_WNLI",
+    "GO_EMOTIONS",
     "GPRO",
-    "HunerEntityLinkingDictionary",
     "HUNER_CELL_LINE",
     "HUNER_CELL_LINE_CELL_FINDER",
     "HUNER_CELL_LINE_CLL",
@@ -404,77 +418,24 @@
     "HUNER_SPECIES_S800",
     "HUNER_SPECIES_VARIOME",
     "IEPA",
+    "IMDB",
     "JNLPBA",
+    "KEYPHRASE_INSPEC",
+    "KEYPHRASE_SEMEVAL2010",
+    "KEYPHRASE_SEMEVAL2017",
     "LINNEAUS",
     "LOCTEXT",
+    "MASAKHA_POS",
     "MIRNA",
+    "NCBI_DISEASE",
     "NCBI_GENE_HUMAN_DICTIONARY",
     "NCBI_TAXONOMY_DICTIONARY",
-    "CTD_DISEASES_DICTIONARY",
-    "CTD_CHEMICALS_DICTIONARY",
-    "NCBI_DISEASE",
-    "ONTONOTES",
-    "OSIRIS",
-    "PDR",
-    "S800",
-    "SCAI_CHEMICALS",
-    "SCAI_DISEASE",
-    "VARIOME",
-    "AMAZON_REVIEWS",
-    "COMMUNICATIVE_FUNCTIONS",
-    "GERMEVAL_2018_OFFENSIVE_LANGUAGE",
-    "GLUE_COLA",
-    "GO_EMOTIONS",
-    "IMDB",
-    "NEWSGROUPS",
-    "STACKOVERFLOW",
-    "SENTEVAL_CR",
-    "SENTEVAL_MPQA",
-    "SENTEVAL_MR",
-    "SENTEVAL_SST_BINARY",
-    "SENTEVAL_SST_GRANULAR",
-    "SENTEVAL_SUBJ",
-    "SENTIMENT_140",
-    "TREC_6",
-    "TREC_50",
-    "WASSA_ANGER",
-    "WASSA_FEAR",
-    "WASSA_JOY",
-    "WASSA_SADNESS",
-    "YAHOO_ANSWERS",
-    "ClassificationCorpus",
-    "ClassificationDataset",
-    "CSVClassificationCorpus",
-    "CSVClassificationDataset",
     "NEL_ENGLISH_AIDA",
     "NEL_ENGLISH_AQUAINT",
     "NEL_ENGLISH_IITB",
     "NEL_ENGLISH_REDDIT",
     "NEL_ENGLISH_TWEEKI",
     "NEL_GERMAN_HIPE",
-    "WSD_MASC",
-    "WSD_OMSTI",
-    "WSD_RAGANATO_ALL",
-    "WSD_SEMCOR",
-    "WSD_TRAINOMATIC",
-    "WSD_UFSAC",
-    "WSD_WORDNET_GLOSS_TAGGED",
-    "RE_ENGLISH_CONLL04",
-    "RE_ENGLISH_DRUGPROT",
-    "RE_ENGLISH_SEMEVAL2010",
-    "RE_ENGLISH_TACRED",
-    "BIOSCOPE",
-    "CONLL_03",
-    "CONLL_03_DUTCH",
-    "CONLL_03_GERMAN",
-    "CONLL_03_SPANISH",
-    "CLEANCONLL",
-    "CONLL_2000",
-    "FEWNERD",
-    "KEYPHRASE_INSPEC",
-    "KEYPHRASE_SEMEVAL2010",
-    "KEYPHRASE_SEMEVAL2017",
-    "MASAKHA_POS",
     "NER_ARABIC_ANER",
     "NER_ARABIC_AQMAR",
     "NER_BASQUE",
@@ -491,6 +452,7 @@
     "NER_ENGLISH_WEBPAGES",
     "NER_ENGLISH_WIKIGOLD",
     "NER_ENGLISH_WNUT_2020",
+    "NER_ESTONIAN_NOISY",
     "NER_FINNISH",
     "NER_GERMAN_BIOFID",
     "NER_GERMAN_EUROPARL",
@@ -499,51 +461,44 @@
     "NER_GERMAN_MOBIE",
     "NER_GERMAN_POLITICS",
     "NER_HIPE_2022",
-    "NER_NOISEBENCH",
     "NER_HUNGARIAN",
     "NER_ICDAR_EUROPEANA",
     "NER_ICELANDIC",
     "NER_JAPANESE",
-    "NER_NERMUD",
     "NER_MASAKHANE",
+    "NER_MULTI_CONER",
+    "NER_MULTI_CONER_V2",
     "NER_MULTI_WIKIANN",
     "NER_MULTI_WIKINER",
     "NER_MULTI_XTREME",
+    "NER_NERMUD",
+    "NER_NOISEBENCH",
     "NER_SWEDISH",
     "NER_TURKU",
     "NER_UKRAINIAN",
-    "NER_ESTONIAN_NOISY",
-    "UP_CHINESE",
-    "UP_ENGLISH",
-    "UP_FINNISH",
-    "UP_FRENCH",
-    "UP_GERMAN",
-    "UP_ITALIAN",
-    "UP_SPANISH",
-    "UP_SPANISH_ANCORA",
-    "WNUT_17",
-    "ColumnCorpus",
-    "ColumnDataset",
-    "NER_MULTI_CONER",
-    "NER_MULTI_CONER_V2",
-    "FeideggerCorpus",
-    "FeideggerDataset",
-    "GLUE_MNLI",
-    "GLUE_MRPC",
-    "GLUE_QNLI",
-    "GLUE_QQP",
-    "GLUE_RTE",
-    "GLUE_WNLI",
-    "GLUE_SST2",
-    "GLUE_STSB",
+    "NEWSGROUPS",
+    "ONTONOTES",
+    "OSIRIS",
+    "PDR",
+    "RE_ENGLISH_CONLL04",
+    "RE_ENGLISH_DRUGPROT",
+    "RE_ENGLISH_SEMEVAL2010",
+    "RE_ENGLISH_TACRED",
+    "S800",
+    "SCAI_CHEMICALS",
+    "SCAI_DISEASE",
+    "SENTEVAL_CR",
+    "SENTEVAL_MPQA",
+    "SENTEVAL_MR",
+    "SENTEVAL_SST_BINARY",
+    "SENTEVAL_SST_GRANULAR",
+    "SENTEVAL_SUBJ",
+    "SENTIMENT_140",
+    "SROIE",
+    "STACKOVERFLOW",
     "SUPERGLUE_RTE",
-    "DataPairCorpus",
-    "DataPairDataset",
-    "DataTripleCorpus",
-    "DataTripleDataset",
-    "OpusParallelCorpus",
-    "ParallelTextCorpus",
-    "ParallelTextDataset",
+    "TREC_6",
+    "TREC_50",
     "UD_AFRIKAANS",
     "UD_ANCIENT_GREEK",
     "UD_ARABIC",
@@ -603,7 +558,52 @@
     "UD_TURKISH",
     "UD_UKRAINIAN",
     "UD_WOLOF",
+    "UP_CHINESE",
+    "UP_ENGLISH",
+    "UP_FINNISH",
+    "UP_FRENCH",
+    "UP_GERMAN",
+    "UP_ITALIAN",
+    "UP_SPANISH",
+    "UP_SPANISH_ANCORA",
+    "VARIOME",
+    "WASSA_ANGER",
+    "WASSA_FEAR",
+    "WASSA_JOY",
+    "WASSA_SADNESS",
+    "WNUT_17",
+    "WSD_MASC",
+    "WSD_OMSTI",
+    "WSD_RAGANATO_ALL",
+    "WSD_SEMCOR",
+    "WSD_TRAINOMATIC",
+    "WSD_UFSAC",
+    "WSD_WORDNET_GLOSS_TAGGED",
+    "YAHOO_ANSWERS",
+    "ZELDA",
+    "CSVClassificationCorpus",
+    "CSVClassificationDataset",
+    "ClassificationCorpus",
+    "ClassificationDataset",
+    "ColumnCorpus",
+    "ColumnDataset",
+    "DataLoader",
+    "DataPairCorpus",
+    "DataPairDataset",
+    "DataTripleCorpus",
+    "DataTripleDataset",
+    "EntityLinkingDictionary",
+    "FeideggerCorpus",
+    "FeideggerDataset",
+    "FlairDatapointDataset",
+    "HunerEntityLinkingDictionary",
+    "MongoDataset",
+    "OcrJsonDataset",
+    "OpusParallelCorpus",
+    "ParallelTextCorpus",
+    "ParallelTextDataset",
+    "SentenceDataset",
+    "StringDataset",
     "UniversalDependenciesCorpus",
     "UniversalDependenciesDataset",
-    "ZELDA",
 ]

flair/datasets/biomedical.py

@@ -1104,7 +1104,7 @@ def parse_file(cls, input_file: Path, split: str, sentence_separator: str) -> In
                     document_text += sentence_separator
 
                 sentence_offset = len(document_text)
-                document_text += sentence.get("text") if document_text else sentence.get("text")
+                document_text += sentence.get("text")
 
                 for entity in sentence.xpath(".//entity"):
                     start, end = entity.get("charOffset").split("-")
@@ -4146,7 +4146,7 @@ def download_corpus(download_folder: Path) -> tuple[Path, Path, Path]:
 
 @deprecated(
     version="0.13",
-    reason='Please use BIGBIO_NER_CORPUS implementation by calling ´corpus = BIGBIO_NER_CORPUS("bigbio/anat_em", trust_remote_code=True)´',
+    reason='Please use BIGBIO_NER_CORPUS implementation by calling `corpus = BIGBIO_NER_CORPUS("bigbio/anat_em", trust_remote_code=True)`',
 )
 class ANAT_EM(ColumnCorpus):
     """Corpus for anatomical named entity mention recognition.
@@ -4157,8 +4157,6 @@ class ANAT_EM(ColumnCorpus):
     http://nactem.ac.uk/anatomytagger/#AnatEM
     """
 
-    pass
-
 
 class BioBertHelper(ColumnCorpus):
     """Helper class to convert corpora and the respective train, dev and test split used by BioBERT.