Models container, created first iteration container supporting classification models

Author: wtsimple

models_container.py

@@ -0,0 +1,47 @@
+from enum import Enum
+
+from pyspark.ml.classification import LogisticRegression, RandomForestClassifier, GBTClassifier, \
+    LinearSVC, NaiveBayes
+
+from spark_launcher import SparkLauncher
+
+
+class ModelKinds(Enum):
+    CLASSIFICATION = 'classification'
+    REGRESSION = 'regression'
+
+
+class Model(object):
+    def __init__(self, model, name='', kind=ModelKinds.CLASSIFICATION):
+        self.model = model
+        self.name = name
+        self.kind = kind
+
+
+class ModelsContainer(object):
+    model_path = "pyspark.ml"
+
+
+    def __init__(self):
+        self.spark = SparkLauncher()
+        # Models
+        self.logistic_class = LogisticRegression(maxIter=20)
+        self.random_forest_class = RandomForestClassifier(cacheNodeIds=True)
+        self.gbt_class = GBTClassifier(cacheNodeIds=True)
+        self.svm_class = LinearSVC(maxIter=20)
+        self.naive_bayes_class = NaiveBayes()
+
+        self._wrap_models()
+
+
+    @property
+    def classification(self):
+        return [obj for name, obj in self.__dict__.items()
+                if getattr(obj, "kind", None) == ModelKinds.CLASSIFICATION]
+
+
+    def _wrap_models(self):
+        for name, obj in self.__dict__.items():
+            if self.model_path in str(obj.__class__):
+                wrapped = Model(model=obj, name=name)
+                setattr(self, name, wrapped)

tests/test_models_container.py

@@ -0,0 +1,14 @@
+from models_container import ModelsContainer
+
+
+def test_example_models_init():
+    ModelsContainer()
+
+
+def test_get_classification_models():
+    names = [model.name for model in ModelsContainer().classification]
+    assert set(names) == {"logistic_class",
+                          "random_forest_class",
+                          "gbt_class",
+                          "svm_class",
+                          "naive_bayes_class"}

todo_list.md

@@ -14,8 +14,14 @@
     - [x] prepare the data frame by applying all transformations 
     (cleaning, encoding, etc)        
 - [x] obtain evaluation metrics for a single model
-- [ ] fit several classification models
-- [ ] compare all classification models
+- [ ] **fit and compare several classification models without tuning**
+    - [ ] create an object container for the models
+    - [ ] initialize the models with default hyperparameters
+    - [ ] fit and compare the results with the evaluator
+- [ ] fit and compare several classification models with tuning and crossvalidation
+    - [ ] be able to pass a list of hyperparameters values for each hyperparameter
+    - [ ] tune and obtain the best hyperparam set per model
+    - [ ] compare the tuned models with the evaluator
 - [ ] prepare data for regression
 - [ ] fit regression model(s)
 - [ ] obtain the regression metrics and compare the models