resolved classification bug in decision tree using n_classes

Author: Shihab-Shahriar

examples/random_forest.py

@@ -1,8 +1,11 @@
+from timeit import default_timer
+start = default_timer()
 import logging
 
-from sklearn.datasets import make_classification
+import numpy as np
+from sklearn.datasets import make_classification, load_boston, load_digits, load_breast_cancer, load_iris
 from sklearn.datasets import make_regression
-from sklearn.metrics import roc_auc_score
+from sklearn.metrics import roc_auc_score, accuracy_score
 
 try:
     from sklearn.model_selection import train_test_split
@@ -20,13 +23,15 @@ def classification():
     X, y = make_classification(
         n_samples=500, n_features=10, n_informative=10, random_state=1111, n_classes=2, class_sep=2.5, n_redundant=0
     )
+    #X,y = load_breast_cancer(return_X_y=True)   
 
     X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.15, random_state=1111)
 
-    model = RandomForestClassifier(n_estimators=10, max_depth=4)
+    model = RandomForestClassifier(n_estimators=5, max_depth=4)
     model.fit(X_train, y_train)
-    predictions = model.predict(X_test)[:, 1]
-    # print(predictions)
+    predictions = model.predict(X_test)[:,1]
+    #predictions = np.argmax(model.predict(X_test),axis=1)
+    print(predictions.shape)
     print("classification, roc auc score: %s" % roc_auc_score(y_test, predictions))
 
 
@@ -46,3 +51,5 @@ def regression():
 if __name__ == "__main__":
     classification()
     # regression()
+    end = default_timer()
+    print(end-start)

mla/ensemble/random_forest.py

@@ -39,13 +39,15 @@ def fit(self, X, y):
         self._train()
 
     def _train(self):
+        n_classes = None if self.trees[0].regression else len(np.unique(self.y))
         for tree in self.trees:
             tree.train(
                 self.X,
                 self.y,
                 max_features=self.max_features,
                 min_samples_split=self.min_samples_split,
                 max_depth=self.max_depth,
+                n_classes=n_classes
             )
 
     def _predict(self, X=None):
@@ -78,10 +80,14 @@ def _predict(self, X=None):
         for i in range(X.shape[0]):
             row_pred = np.zeros(y_shape)
             for tree in self.trees:
-                row_pred += tree.predict_row(X[i, :])
+                tmp = tree.predict_row(X[i, :])
+                print(tmp,row_pred.shape,row_pred)
+                row_pred += tmp
+
 
             row_pred /= self.n_estimators
             predictions[i, :] = row_pred
+            print(f"i={i},{row_pred}\n")
         return predictions
 
 

mla/ensemble/tree.py

@@ -64,7 +64,8 @@ def _find_best_split(self, X, target, n_features):
                     max_col, max_val, max_gain = column, value, gain
         return max_col, max_val, max_gain
 
-    def train(self, X, target, max_features=None, min_samples_split=10, max_depth=None, minimum_gain=0.01, loss=None):
+    def train(self, X, target, max_features=None, min_samples_split=10, max_depth=None, 
+                minimum_gain=0.01, loss=None, n_classes = None):
         """Build a decision tree from training set.
 
         Parameters
@@ -84,6 +85,8 @@ def train(self, X, target, max_features=None, min_samples_split=10, max_depth=No
             Minimum gain required for splitting.
         loss : function, default None
             Loss function for gradient boosting.
+        n_classes : int, default None
+            No of unique labels in case of classification
         """
 
         if not isinstance(target, dict):
@@ -118,17 +121,17 @@ def train(self, X, target, max_features=None, min_samples_split=10, max_depth=No
             # Grow left and right child
             self.left_child = Tree(self.regression, self.criterion)
             self.left_child.train(
-                left_X, left_target, max_features, min_samples_split, max_depth - 1, minimum_gain, loss
+                left_X, left_target, max_features, min_samples_split, max_depth - 1, minimum_gain, loss, n_classes
             )
 
             self.right_child = Tree(self.regression, self.criterion)
             self.right_child.train(
-                right_X, right_target, max_features, min_samples_split, max_depth - 1, minimum_gain, loss
+                right_X, right_target, max_features, min_samples_split, max_depth - 1, minimum_gain, loss, n_classes
             )
         except AssertionError:
-            self._calculate_leaf_value(target)
+            self._calculate_leaf_value(target, n_classes)
 
-    def _calculate_leaf_value(self, targets):
+    def _calculate_leaf_value(self, targets, n_classes):
         """Find optimal value for leaf."""
         if self.loss is not None:
             # Gradient boosting
@@ -140,7 +143,8 @@ def _calculate_leaf_value(self, targets):
                 self.outcome = np.mean(targets["y"])
             else:
                 # Probability for classification task
-                self.outcome = stats.itemfreq(targets["y"])[:, 1] / float(targets["y"].shape[0])
+                #self.outcome = stats.itemfreq(targets["y"])[:, 1] / float(targets["y"].shape[0])
+                self.outcome = np.bincount(targets["y"], minlength=n_classes) / targets["y"].shape[0]
 
     def predict_row(self, row):
         """Predict single row."""