Merge pull request #117 from ardunn/master

Adding top level class + bugfixes

Author: ardunn

matbench/automl/adaptors.py

@@ -13,7 +13,7 @@
 
 from matbench.automl.tpot_configs.classifier import classifier_config_dict_mb
 from matbench.automl.tpot_configs.regressor import regressor_config_dict_mb
-from matbench.utils.utils import is_greater_better, MatbenchError
+from matbench.utils.utils import is_greater_better, MatbenchError, set_fitted, check_fitted
 from matbench.base import AutoMLAdaptor, LoggableMixin
 
 __authors__ = ['Alex Dunn <[email protected]'
@@ -92,7 +92,7 @@ def __init__(self, mode, logger=True, **tpot_kwargs):
         self.is_fit = False
         self.random_state = tpot_kwargs.get('random_state', None)
 
-
+    @set_fitted
     def fit(self, df, target, **fit_kwargs):
         """
         Train a TPOTRegressor or TPOTClassifier by fitting on a dataframe.
@@ -112,14 +112,14 @@ def fit(self, df, target, **fit_kwargs):
         X = df.drop(columns=target).values.tolist()
         self._features = df.drop(columns=target).columns.tolist()
         self._ml_data = {"X": X, "y": y}
-        self.is_fit = True
         self.fitted_target = target
         self.logger.info("TPOT fitting started.")
         self._backend = self._backend.fit(X, y, **fit_kwargs)
         self.logger.info("TPOT fitting finished.")
         return self
 
 
+    @check_fitted
     @property
     def _best_models(self):
         """
@@ -134,9 +134,6 @@ def _best_models(self):
                 best hyperparameter combination found.
 
         """
-        if not self.is_fit:
-            raise NotFittedError("Error, the model has not yet been fit")
-
         self.greater_score_is_better = is_greater_better(
             self.backend.scoring_function)
 
@@ -179,6 +176,7 @@ def _best_models(self):
         self.models = models
         return best_models_and_scores
 
+    @check_fitted
     def predict(self, df, target):
         """
         Predict the target property of materials given a df of features.
@@ -231,8 +229,8 @@ def predict(self, df, target):
 
     # Load a dataset
     df = load_dataset("elastic_tensor_2015").rename(columns={"formula": "composition"})[["composition",  "K_VRH"]]
-    testdf = df.iloc[60:90]
-    traindf = df.iloc[:500]
+    testdf = df.iloc[501:550]
+    traindf = df.iloc[:100]
     target = "K_VRH"
 
     # Get top-lvel transformers
@@ -245,13 +243,11 @@ def predict(self, df, target):
     traindf = autofeater.fit_transform(traindf, target)
     traindf = cleaner.fit_transform(traindf, target)
     traindf = reducer.fit_transform(traindf, target)
+    learner.fit(traindf, target)
 
     # Use transformers on testing data
     testdf = autofeater.transform(testdf, target)
     testdf = cleaner.transform(testdf, target)
     testdf = reducer.transform(testdf, target)
-
-    # Use training data to predict testing data
-    learner.fit(traindf, target)
     testdf = learner.predict(testdf, target)
     print(testdf)

matbench/base.py

@@ -3,6 +3,7 @@
 """
 
 import logging
+
 from matbench.utils.utils import initialize_logger, initialize_null_logger
 
 __authors__ = ["Alex Dunn <[email protected]>", "Alex Ganose <[email protected]>"]

matbench/featurization/core.py

@@ -2,7 +2,7 @@
 from pymatgen import Composition
 from matminer.featurizers.conversions import StructureToOxidStructure, StrToComposition, DictToObject, StructureToComposition
 
-from matbench.utils.utils import MatbenchError
+from matbench.utils.utils import MatbenchError, check_fitted, set_fitted
 from matbench.base import DataframeTransformer, LoggableMixin
 from matbench.featurization.sets import CompositionFeaturizers, \
     StructureFeaturizers, BSFeaturizers, DOSFeaturizers
@@ -197,6 +197,7 @@ def __init__(self, featurizers=None, ignore_cols=None, ignore_errors=True,
                         featurizers[ftype] = []
                 self.featurizers = featurizers
 
+    @set_fitted
     def fit(self, df, target):
         """
         Fit all featurizers to the df.
@@ -219,7 +220,6 @@ def fit(self, df, target):
         Returns:
             (AutoFeaturizer): self
         """
-        self.is_fit = False
         df = self._prescreen_df(df, inplace=True)
         df = self._add_composition_from_structure(df)
         for featurizer_type, featurizers in self.featurizers.items():
@@ -234,9 +234,9 @@ def fit(self, df, target):
                     self.features += f.feature_labels()
                     self.logger.info("Fit {} to {} samples in dataframe."
                                      "".format(f.__class__.__name__, df.shape[0]))
-        self.is_fit = True
         return self
 
+    @check_fitted
     def transform(self, df, target):
         """
         Decorate a dataframe containing composition, structure, bandstructure,
@@ -249,9 +249,6 @@ def transform(self, df, target):
         Returns:
             df (pandas.DataFrame): Transformed dataframe containing features.
         """
-        if not self.is_fit:
-            # Featurization requires featurizers already be fit...
-            raise NotFittedError("AutoFeaturizer has not been fit!")
         df = self._prescreen_df(df, inplace=True)
         df = self._add_composition_from_structure(df)
 
@@ -369,6 +366,7 @@ def _add_composition_from_structure(self, df):
                 df = struct2comp.featurize_dataframe(df, "structure")
         return df
 
+
 if __name__ == "__main__":
     from matminer.datasets.dataset_retrieval import load_dataset
     df = load_dataset("flla")

matbench/featurization/tests/test_core.py

@@ -6,19 +6,24 @@
 from pymatgen import Composition
 from matminer.data_retrieval.retrieve_MP import MPDataRetrieval
 from matminer.datasets.dataset_retrieval import load_dataset
-from matminer.featurizers.composition import ElectronAffinity, ElementProperty, AtomicOrbitals
-from matminer.featurizers.structure import GlobalSymmetryFeatures, DensityFeatures
+from matminer.featurizers.composition import ElectronAffinity, ElementProperty, \
+    AtomicOrbitals
+from matminer.featurizers.structure import GlobalSymmetryFeatures, \
+    DensityFeatures
 
 from matbench.featurization.core import AutoFeaturizer
 
 test_dir = os.path.dirname(__file__)
 
-__author__ = ["Alex Dunn <[email protected]>", "Alireza Faghaninia <[email protected]>"]
+__author__ = ["Alex Dunn <[email protected]>",
+              "Alireza Faghaninia <[email protected]>"]
+
 
 class TestAutoFeaturizer(unittest.TestCase):
 
     def setUp(self, limit=5):
-        self.test_df = load_dataset('elastic_tensor_2015').rename(columns={"formula": "composition"})
+        self.test_df = load_dataset('elastic_tensor_2015').rename(
+            columns={"formula": "composition"})
         self.limit = limit
 
     def test_sanity(self):
@@ -53,7 +58,6 @@ def test_featurize_composition(self):
         self.assertEqual(df["LUMO_element"].iloc[0], "Nb")
         self.assertTrue("composition" not in df.columns)
 
-
     def test_featurize_structure(self):
         """
         Test automatic featurization while only considering structure.
@@ -120,7 +124,6 @@ def test_exclusions(self):
         for flabels in [ep_feats, ef_feats, ao_feats]:
             self.assertFalse(any([f in df.columns for f in flabels]))
 
-
     def test_featurize_bsdos(self, refresh_df_init=False, limit=1):
         """
         Tests featurize_dos and featurize_bandstructure.
@@ -138,11 +141,11 @@ def test_featurize_bsdos(self, refresh_df_init=False, limit=1):
         if refresh_df_init:
             mpdr = MPDataRetrieval()
             df = mpdr.get_dataframe(criteria={"material_id": "mp-149"},
-                                         properties=["pretty_formula",
-                                                     "dos",
-                                                     "bandstructure",
-                                                     "bandstructure_uniform"]
-                                         )
+                                    properties=["pretty_formula",
+                                                "dos",
+                                                "bandstructure",
+                                                "bandstructure_uniform"]
+                                    )
             df.to_pickle(os.path.join(test_dir, df_bsdos_pickled))
         else:
             df = pd.read_pickle(os.path.join(test_dir, df_bsdos_pickled))