Model evaluator, produces right metrics for single classification model

Author: wtsimple

data_preprocessor.py

@@ -15,6 +15,9 @@ def __init__(self, train_df: DataFrame = None, test_df: DataFrame = None):
         self.train_encoded_df = None
         self.test_encoded_df = None
 
+        self.one_hot_suffix = '_vec'
+        self.indexed_suffix = "_cat"
+
 
     def explore_factors(self):
         """Generates a dictionary of one pandas dataframe per column
@@ -88,22 +91,33 @@ def assemble_features(self, *columns, out_name='features'):
     def prepare_to_model(self, target_col: str, to_strip=' '):
         """Runs all cleaning and encoding steps to generate
         dataframes ready to use in modeling"""
+        # if target_col in self.factors:
+        #     target_col += indexed_suffix
         self.strip_columns(*self.factors, to_strip=to_strip)
-        self.string_index(*self.factors, suffix='_cat')
+        self.string_index(*self.factors, suffix=self.indexed_suffix)
         # one-hot encode indexed factors, except target
-        to_one_hot_encode = [fac + "_cat" for fac in self.factors if fac != target_col]
-        self.one_hot_encode(*to_one_hot_encode, suffix='_vec')
+        self.one_hot_encode(*self._one_hot_encode_columns(target_col), suffix=self.one_hot_suffix)
         # assemble all together with numeric columns into features (except target if it's numeric)
-        to_assemble = [col for col in self.numeric_columns if col != target_col]
-        to_assemble += [col for col, data_type in self.train_df.dtypes if "_cat_vec" in col]
-        self.assemble_features(*to_assemble)
-        if target_col in self.factors:
-            target_col += "_cat"
+        self.assemble_features(*self._columns_to_assemble(target_col))
 
+        if target_col in self.factors:
+            target_col += self.indexed_suffix
         self.train_encoded_df = self._select_to_model(self.train_df, target_col)
         self.test_encoded_df = self._select_to_model(self.test_df, target_col)
 
 
+    def _one_hot_encode_columns(self, target_col):
+        return [fac + self.indexed_suffix for fac in self.factors if fac != target_col]
+
+
+    def _columns_to_assemble(self, target_col):
+        numeric = [col for col in self.numeric_columns
+                   if col != target_col and not col.endswith(self.indexed_suffix)]
+        one_hot_encoded = [col for col, data_type in self.train_df.dtypes
+                           if self.indexed_suffix + self.one_hot_suffix in col]
+        return numeric + one_hot_encoded
+
+
     @property
     def factors(self) -> List[str]:
         return self._get_cols_by_types(types=['string'])

model_evaluator.py

@@ -14,12 +14,21 @@ def __init__(self, metrics_class=None):
         self.metrics_class = metrics_class if metrics_class else BinaryClassificationMetrics
 
 
-    @staticmethod
-    def compare(data_frames: Dict[str, DataFrame], models: list):
-        output_value = 1.0
-        df = data_frames[list(data_frames.keys())[0]]
-        if len(df.take(20)) > 10:
-            output_value = 0.768680
+    def compare(self, data_frames: Dict[str, DataFrame], models: list):
+        # per model and per data frame calculate all the metrics
+        index = []
         metrics = ["areaUnderROC", "areaUnderPR"]
-        return pandas.DataFrame({metric: [output_value for m in metrics] for metric in metrics},
-                                index=[key for key in data_frames.keys()])
+        data = {metric: [] for metric in metrics}
+        for model in models:
+            for df_name, df in data_frames.items():
+                index.append(self.index_key(df_name, model))
+                evaluator = self.metrics_class(model.transform(df).select('prediction', 'label').rdd)
+                for metric in metrics:
+                    data[metric].append(getattr(evaluator, metric))
+
+        return pandas.DataFrame(data, index=index)
+
+
+    @staticmethod
+    def index_key(df_name, model):
+        return model.__class__.__name__ + "_" + df_name

tests/conftest.py

@@ -34,6 +34,6 @@ def _get_preprocessor(data_path: str):
 
 def _fit_logistic(preprocessor):
     preprocessor.prepare_to_model(target_col='income', to_strip=' .')
-    lr = LogisticRegression(maxIter=10, regParam=0.1, elasticNetParam=0.2)
+    lr = LogisticRegression(maxIter=10, regParam=0, elasticNetParam=0)
     fit_model = lr.fit(preprocessor.train_encoded_df)
     return fit_model

tests/test_model_evaluator.py

@@ -22,7 +22,8 @@ def test_model_evaluator_with_linear_regression(logistic_model, preprocessor):
 
 def test_model_evaluator_with_linear_regression_and_full_train_data(logistic_model_train_data, preprocessor_train_data):
     _check_evaluation(preprocessor=preprocessor_train_data,
-                      model=logistic_model_train_data, metrics={"areaUnderROC": 0.768680, "areaUnderPR": 0.640418})
+                      model=logistic_model_train_data,
+                      metrics={"areaUnderROC": 0.764655781, "areaUnderPR": 0.63384702449})
 
 
 def _check_evaluation(preprocessor, model, metrics: Dict[str, float]):
@@ -39,4 +40,4 @@ def _check_evaluation(preprocessor, model, metrics: Dict[str, float]):
         for metric in metrics:
             assert metric in comparison
             for dataframe in dataframes:
-                assert comparison[metric][dataframe] == pytest.approx(metrics[metric])
+                assert comparison[metric][evaluator.index_key(dataframe, model)] == pytest.approx(metrics[metric])

todo_list.md

@@ -13,7 +13,7 @@
 - [x] fit a simple classification model
     - [x] prepare the data frame by applying all transformations 
     (cleaning, encoding, etc)        
-- [ ] **obtain evaluation metrics for the model**
+- [x] obtain evaluation metrics for a single model
 - [ ] fit several classification models
 - [ ] compare all classification models
 - [ ] prepare data for regression