derive-features-internal-and-external-validation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import warnings filter\n",
    "from warnings import simplefilter\n",
    "# ignore all future warnings\n",
    "simplefilter(action='ignore', category=FutureWarning)\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from numpy import arange\n",
    "from numpy import argmax\n",
    "import xgboost\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.ensemble import StackingClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from numpy import arange\n",
    "from sklearn.metrics import accuracy_score\n",
    "import copy\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from numpy import argmax\n",
    "from sklearn import metrics\n",
    "from imblearn.metrics import sensitivity_score\n",
    "from imblearn.metrics import specificity_score\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import LinearSVC, SVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier, VotingClassifier\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import MinMaxScaler, RobustScaler\n",
    "import pickle\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.utils import resample\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.metrics import classification_report, balanced_accuracy_score\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import numpy as np\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
    "import scipy.stats as ss\n",
    "import itertools\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "from sklearn.metrics import fbeta_score, make_scorer\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from sklearn.model_selection import LeaveOneOut\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import KFold\n",
    "import scipy\n",
    "import numpy as np, scipy.stats as st\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from scipy import interp\n",
    "from collections import Counter\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from imblearn.ensemble import BalancedBaggingClassifier\n",
    "from imblearn.ensemble import BalancedRandomForestClassifier\n",
    "from imblearn.ensemble import EasyEnsembleClassifier\n",
    "from imblearn.ensemble import RUSBoostClassifier\n",
    "from imblearn.under_sampling import ClusterCentroids \n",
    "from sklearn.linear_model import RidgeCV, LassoCV, Ridge, Lasso\n",
    "from imblearn.under_sampling import RepeatedEditedNearestNeighbours\n",
    "from imblearn.under_sampling import EditedNearestNeighbours\n",
    "from imblearn.under_sampling import RepeatedEditedNearestNeighbours\n",
    "from sklearn.linear_model import Perceptron\n",
    "\n",
    "from imblearn.under_sampling import AllKNN\n",
    "from imblearn.under_sampling import CondensedNearestNeighbour\n",
    "from imblearn.under_sampling import TomekLinks\n",
    "from imblearn.under_sampling import InstanceHardnessThreshold\n",
    "from imblearn.under_sampling import NearMiss\n",
    "from imblearn.under_sampling import NeighbourhoodCleaningRule\n",
    "from imblearn.under_sampling import     OneSidedSelection \n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.metrics import classification_report_imbalanced\n",
    "from imblearn.under_sampling import ClusterCentroids\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.over_sampling import ADASYN\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from imblearn.over_sampling import SVMSMOTE\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "from imblearn.over_sampling import BorderlineSMOTE\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.over_sampling import SVMSMOTE\n",
    "from imblearn.over_sampling import SMOTENC\n",
    "from imblearn.over_sampling import ADASYN\n",
    "from imblearn.over_sampling import KMeansSMOTE\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from imblearn.over_sampling import BorderlineSMOTE\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.over_sampling import SVMSMOTE\n",
    "from imblearn.over_sampling import SMOTENC\n",
    "from imblearn.over_sampling import ADASYN\n",
    "from imblearn.over_sampling import KMeansSMOTE\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from imblearn.over_sampling import BorderlineSMOTE\n",
    "import itertools\n",
    "from imblearn.metrics import sensitivity_score\n",
    "from imblearn.metrics import specificity_score\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "import random\n",
    "import pymrmr\n",
    "from sklearn.feature_selection import SequentialFeatureSelector\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "from sklearn.linear_model import RidgeCV\n",
    "from sklearn.feature_selection import f_classif\n",
    "from sklearn.feature_selection import *\n",
    "from yellowbrick.model_selection import rfecv\n",
    "from collections import defaultdict\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Do not use any sampling technique to balance the dataset - its already balanced (more or less). Repeatedly split the dataset into train and test and select top features using RFECV. Then use the test set to tune hyperparameters - find the best classifier and threshold. Then take the best model for the internal validation test set and apply it on 20271 and 20194. Try ensemble models also"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "scorer = make_scorer(matthews_corrcoef)\n",
    "def model_tune_nb(X_samp, y_samp):\n",
    "    nb = GaussianNB()\n",
    "    grid_params = [{'var_smoothing': [1e-11, 1e-10, 1e-9]}]\n",
    "    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    clf = GridSearchCV(estimator=nb, param_grid=grid_params, scoring=scorer, cv=cv, \n",
    "                               refit=True, return_train_score=True, n_jobs=100)\n",
    "    clf.fit(X_samp,y_samp)\n",
    "    print(\"Tuning Done\")\n",
    "    best_model = clf.best_estimator_\n",
    "    best_model.fit(X_samp, y_samp)\n",
    "    return best_model,clf.best_score_\n",
    "\n",
    "def model_tune_lda(X_samp, y_samp):\n",
    "    lda = QuadraticDiscriminantAnalysis()\n",
    "    #grid_params = [{'solver' : ['svd', 'lsqr', 'eigen']}]\n",
    "    grid_params = [{'reg_param': [0.1, 0.2, 0.3, 0.4, 0.5]}]\n",
    "    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    clf = GridSearchCV(estimator=lda, param_grid=grid_params, scoring=scorer, cv=cv, \n",
    "                               refit=True, return_train_score=True, n_jobs=100)\n",
    "    clf.fit(X_samp,y_samp)\n",
    "    print(\"Tuning Done\")\n",
    "    best_model = clf.best_estimator_\n",
    "    best_model.fit(X_samp, y_samp)\n",
    "    return best_model,clf.best_score_\n",
    "\n",
    "def model_tune_svm(X_samp, y_samp):\n",
    "    svm = SVC(random_state=42, probability=True, class_weight=\"balanced\")\n",
    "    grid_params= [\n",
    "                  {'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000], 'kernel': ['linear']},\n",
    "                  {'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000], 'gamma': [0.001, 0.0001], 'kernel': ['rbf']},\n",
    "                 ]\n",
    "    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    clf = GridSearchCV(estimator=svm, param_grid=grid_params, scoring=scorer, cv=cv, \n",
    "                               refit=True, return_train_score=True, n_jobs=100)\n",
    "    clf.fit(X_samp,y_samp)\n",
    "    print(\"Tuning Done\")\n",
    "    best_model = clf.best_estimator_\n",
    "    best_model.fit(X_samp, y_samp)\n",
    "    return best_model,clf.best_score_\n",
    "\n",
    "def model_tune_knn(X_samp, y_samp):\n",
    "    knn=KNeighborsClassifier()\n",
    "    grid_params = { 'n_neighbors' : [5,7,9,11,13,15],\n",
    "               'weights' : ['uniform','distance'],\n",
    "               'metric' : ['minkowski','euclidean','manhattan']}\n",
    "    \n",
    "    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    clf = GridSearchCV(estimator=knn, param_grid=grid_params, scoring=scorer, cv=cv, \n",
    "                               refit=True, return_train_score=True, n_jobs=100)\n",
    "    clf.fit(X_samp,y_samp)\n",
    "    print(\"Tuning Done\")\n",
    "    best_model = clf.best_estimator_\n",
    "    best_model.fit(X_samp, y_samp)\n",
    "    return best_model,clf.best_score_\n",
    "\n",
    "def model_tune_logreg(X_samp, y_samp):\n",
    "    logreg = LogisticRegression(class_weight=\"balanced\", random_state=42)\n",
    "    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    grid={\"C\":np.logspace(-3,3,7), \"penalty\":[\"l1\",\"l2\"]}# l1 lasso l2 ridge\n",
    "    clf = GridSearchCV(estimator=logreg, param_grid=grid, scoring=scorer, cv=cv, \n",
    "                               refit=True, return_train_score=True, n_jobs=100)\n",
    "    clf.fit(X_samp,y_samp)\n",
    "    print(\"Tuning done\")\n",
    "    best_model = clf.best_estimator_\n",
    "    best_model.fit(X_samp,y_samp)\n",
    "    return best_model,clf.best_score_\n",
    "\n",
    "def model_tune_brf(X_samp, y_samp):\n",
    "    brf=BalancedRandomForestClassifier(random_state=42, class_weight=\"balanced\")\n",
    "    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    grid_params= { \n",
    "        'n_estimators':[10,20,21,30,50,100,200,500],\n",
    "        'max_depth':[2,4,6,8,10],\n",
    "                   'max_features': ['sqrt', 'log2'],\n",
    "        'max_depth' : [2,3,5,7,9,11,13,15,17,19,21],\n",
    "        'min_samples_leaf': [1,3,5],\n",
    "        'min_samples_split': [2, 5,10,12],\n",
    "    }\n",
    "    clf = GridSearchCV(estimator=brf, param_grid=grid_params, scoring=scorer, cv=cv, \n",
    "                               refit=True, return_train_score=True, n_jobs=100)\n",
    "    \n",
    "    clf.fit(X_samp,y_samp)\n",
    "    print(\"Tuning Done\")\n",
    "    best_model = clf.best_estimator_\n",
    "    print(clf.best_params_)\n",
    "    best_model.fit(X_samp, y_samp)\n",
    "    return best_model,clf.best_score_\n",
    "\n",
    "\n",
    "def model_tune_ee(X_samp, y_samp):\n",
    "    ee=EasyEnsembleClassifier(random_state=42)\n",
    "    grid_params= { \n",
    "        'n_estimators':[10,20,21,30,50,100,200,500],\n",
    "        }\n",
    "    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    clf = GridSearchCV(estimator=ee, param_grid=grid_params, scoring=scorer, cv=cv, \n",
    "                               refit=True, return_train_score=True, n_jobs=100)\n",
    "    clf.fit(X_samp,y_samp)\n",
    "    print(\"Tuning Done\")\n",
    "    best_model = clf.best_estimator_\n",
    "    best_model.fit(X_samp, y_samp)\n",
    "    return best_model,clf.best_score_\n",
    "\n",
    "\n",
    "def model_tune_bagging(X_samp, y_samp):\n",
    "    ee=BalancedBaggingClassifier(random_state=42)\n",
    "    grid_params= { \n",
    "        'n_estimators':[10,20,21,30,50,100,200,500],'max_samples' : [0.05, 0.1, 0.2, 0.5],\n",
    "        }\n",
    "    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    clf = GridSearchCV(estimator=ee, param_grid=grid_params, scoring=scorer, cv=cv, \n",
    "                               refit=True, return_train_score=True, n_jobs=100)\n",
    "    clf.fit(X_samp,y_samp)\n",
    "    print(\"Tuning Done\")\n",
    "    best_model = clf.best_estimator_\n",
    "    best_model.fit(X_samp, y_samp)\n",
    "    return best_model,clf.best_score_\n",
    "\n",
    "def model_tune_dt(X_samp, y_samp):\n",
    "    dt=DecisionTreeClassifier(random_state=42, class_weight=\"balanced\")\n",
    "    grid_params = {'max_depth':[2,4,6,8,10,12],\n",
    "                   'min_samples_split' :np.linspace(0.1, 1.0, 10, endpoint=True),\n",
    "                   'min_samples_leaf' : np.linspace(0.1, 0.5, 5, endpoint=True),\n",
    "                   'max_features' :[None, 'sqrt', 'log2'],\n",
    "                   'criterion': [\"gini\", \"entropy\"]}\n",
    "    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    clf = GridSearchCV(estimator=dt, param_grid=grid_params, scoring=scorer, cv=cv, \n",
    "                               refit=True, return_train_score=True, n_jobs=100)\n",
    "    clf.fit(X_samp,y_samp)\n",
    "    print(\"Tuning Done\")\n",
    "    best_model = clf.best_estimator_\n",
    "    best_model.fit(X_samp, y_samp)\n",
    "    return best_model,clf.best_score_\n",
    "\n",
    "def model_tune_rf(X_samp, y_samp):\n",
    "    rf=RandomForestClassifier(random_state=42, class_weight=\"balanced\")\n",
    "    grid_params= { \n",
    "        'n_estimators':[10,20,21,30,50,100,200,500],\n",
    "        'max_depth':[2,4,6,8,10],\n",
    "                   'max_features': ['sqrt', 'log2'],\n",
    "        'max_depth' : [2,3,5,7,9,11,13,15,17,19,21],\n",
    "        'min_samples_leaf': [1,3,5],\n",
    "        'min_samples_split': [2, 5,10,12],\n",
    "    }\n",
    "    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    clf = GridSearchCV(estimator=rf, param_grid=grid_params, scoring=scorer, cv=cv, \n",
    "                               refit=True, return_train_score=True, n_jobs=100)\n",
    "    clf.fit(X_samp,y_samp)\n",
    "    print(\"Tuning Done\")\n",
    "    best_model = clf.best_estimator_\n",
    "    best_model.fit(X_samp, y_samp)\n",
    "    return best_model,clf.best_score_\n",
    "\n",
    "def model_tune_et(X_samp, y_samp):\n",
    "    rf=ExtraTreesClassifier(random_state=42, class_weight=\"balanced\")\n",
    "    grid_params= { \n",
    "        'n_estimators':[5,10,20,21,30,50,100,200,500],\n",
    "        'max_depth':[2,4,6,8,10],\n",
    "                   'max_features': ['sqrt', 'log2'],\n",
    "        'max_depth' : [2,3,5,7,9,11,13,15,17,19,21],\n",
    "        'min_samples_leaf': [1,3,5],\n",
    "        'min_samples_split': [2, 5,10,12],\n",
    "    }\n",
    "    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    clf = GridSearchCV(estimator=rf, param_grid=grid_params, scoring=scorer, cv=cv, \n",
    "                               refit=True, return_train_score=True, n_jobs=100)\n",
    "    clf.fit(X_samp,y_samp)\n",
    "    print(\"Tuning Done\")\n",
    "    best_model = clf.best_estimator_\n",
    "    best_model.fit(X_samp, y_samp)\n",
    "    return best_model,clf.best_score_\n",
    "\n",
    "def model_tune_ada(X_samp, y_samp):\n",
    "    rf=AdaBoostClassifier(random_state=42)\n",
    "    grid_params= {\n",
    "        'n_estimators':[5,10,20,21,30,50,100,200,500],\n",
    "        'learning_rate':[0.1,0.3,0.5,1,3,5,7,9],\n",
    "        'algorithm':['SAMME', 'SAMME.R']\n",
    "    }\n",
    "    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "    clf = GridSearchCV(estimator=rf, param_grid=grid_params, scoring=scorer, cv=cv, \n",
    "                               refit=True, return_train_score=True, n_jobs=100)\n",
    "    clf.fit(X_samp,y_samp)\n",
    "    print(\"Tuning Done\")\n",
    "    best_model = clf.best_estimator_\n",
    "    best_model.fit(X_samp, y_samp)\n",
    "    return best_model, clf.best_score_\n",
    "\n",
    "def convert_to_labels(pos_probs, threshold):\n",
    "    return (pos_probs >= threshold).astype('int')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(170, 61)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Load training data with a relative path\n",
    "train_data = pd.read_table(\"data/GSE25066/GSE25066_processed.txt\")\n",
    "\n",
    "# Convert labels to categorical codes\n",
    "train_labs = train_data[\"Labels\"].astype(\"category\").cat.codes\n",
    "\n",
    "# Drop the \"Labels\" column\n",
    "train_data = train_data.drop(\"Labels\", axis=1)\n",
    "\n",
    "# Load top features using a relative path\n",
    "top_feats = pd.read_table(\"data/Top_features/top_feats_TILs_pooled_sig_list.txt\")\n",
    "\n",
    "# Select only top features from the training data\n",
    "train_data_feats = train_data[top_feats[\"Genes\"]]\n",
    "\n",
    "# Print the shape of the filtered data\n",
    "print(train_data_feats.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import Random\n",
    "\n",
    "def rfecv_ranked_list_generator(classifier_name, X, y, num_splits,num_repeats):\n",
    "    rskf=RepeatedStratifiedKFold(n_splits=num_splits,n_repeats=num_repeats,random_state=42); my_dict = {}; num_feats =[]\n",
    "    i=0; cols =[];auc=[];sensitivity=[];specificity=[];m=[];accuracy=[];cols=[]\n",
    "    #X=train_data_feats; y=train_labs; \n",
    "    min_features_to_select = 1  # Minimum number of features to consider\n",
    "\n",
    "    if classifier_name == \"RF\":\n",
    "        clf = RandomForestClassifier(random_state=42)\n",
    "    elif classifier_name == \"Logit\":\n",
    "        clf = LogisticRegression(random_state=42)\n",
    "    elif classifier_name == \"SVM\":\n",
    "        clf = SVC(kernel=\"linear\", random_state=42)\n",
    "\n",
    "    for train_index,test_index in rskf.split(X,y):\n",
    "        i=i+1\n",
    "        print(i,end=\"\")\n",
    "        X_train,X_test=X.iloc[train_index],X.iloc[test_index]\n",
    "        y_train,y_test=y.iloc[train_index],y.iloc[test_index]\n",
    "        cv = StratifiedKFold(5); \n",
    "        rfecv = RFECV(\n",
    "        estimator=clf,\n",
    "        step=1,\n",
    "        cv=cv,\n",
    "        scoring=make_scorer(matthews_corrcoef),min_features_to_select=min_features_to_select,\n",
    "        n_jobs=-1,\n",
    "    )\n",
    "        rfecv.fit(X_train, y_train)\n",
    "\n",
    "        my_dict[f'list_{i}'] = X_train.columns[rfecv.support_]\n",
    "        num_feats.append(rfecv.n_features_)\n",
    "\n",
    "\n",
    "    element_counts = Counter()\n",
    "\n",
    "    # Iterate through each list in the dictionary and update the element counts\n",
    "    for lst in my_dict.values():\n",
    "        element_counts.update(lst)\n",
    "\n",
    "    # Sort the elements based on their counts in descending order\n",
    "    ranked_list = sorted(element_counts.items(), key=lambda x: x[1], reverse=True)\n",
    "\n",
    "    # Print the ranked list\n",
    "    runs = num_splits * num_repeats\n",
    "    print(\"\\nRanked list of element counts:\")\n",
    "    for element, count in ranked_list:\n",
    "        print(f\"{element}: {count/runs}\")\n",
    "\n",
    "    n_scores = len(rfecv.cv_results_[\"mean_test_score\"])\n",
    "    plt.figure()\n",
    "    plt.xlabel(\"Number of features selected\")\n",
    "    plt.ylabel(\"Mean test mcc\")\n",
    "    plt.errorbar(\n",
    "    range(min_features_to_select, n_scores + min_features_to_select),\n",
    "    rfecv.cv_results_[\"mean_test_score\"],\n",
    "    yerr=rfecv.cv_results_[\"std_test_score\"],)\n",
    "    plt.title(\"Recursive Feature Elimination\")\n",
    "    plt.show()\n",
    "\n",
    "    return ranked_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#recursive feature elimintion with 5-fold CV ran 100 times resulting in 500 unique train-test splits\n",
    "\n",
    "ranked_list_rf = rfecv_ranked_list_generator(\"RF\",train_data_feats, train_labs, 5, 100) \n",
    "ranked_list_logreg = rfecv_ranked_list_generator(\"Logit\",train_data_feats, train_labs, 5, 100)\n",
    "ranked_list_svm = rfecv_ranked_list_generator(\"SVM\",train_data_feats, train_labs, 5, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def ranked_element_counts(dictionary):\n",
    "    # Initialize a Counter to store element counts\n",
    "    element_counts = Counter()\n",
    "\n",
    "    # Iterate through each list in the dictionary and update the element counts\n",
    "    for lst in dictionary.values():\n",
    "        element_counts.update(lst)\n",
    "\n",
    "    # Sort the elements based on their counts in descending order\n",
    "    ranked_list = sorted(element_counts.items(), key=lambda x: x[1], reverse=True)\n",
    "\n",
    "    return ranked_list\n",
    "\n",
    "ranked_list = ranked_element_counts(my_dict)\n",
    "\n",
    "# Print the ranked list\n",
    "print(\"Ranked list of element counts:\")\n",
    "for element, count in ranked_list:\n",
    "    print(f\"{element}: {count/200}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 457,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import name\n",
    "#ranked_list_rf = pd.read_table('ranked_list_rf.txt')\n",
    "ranked_list_logreg_1 = [item[0] for item in ranked_list_lr]\n",
    "ranked_list_svm_1 = [item[0] for item in ranked_list_svm]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 449,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "def combine_rankings(list_of_lists):\n",
    "    # Initialize dictionary to store total points for each gene\n",
    "    points = defaultdict(int)\n",
    "    num_genes = len(list_of_lists[0])\n",
    "\n",
    "    # Assign points based on rank in each list using Borda Count\n",
    "    for ranked_list in list_of_lists:\n",
    "        for rank, gene in enumerate(ranked_list, 1):\n",
    "            points[gene] += num_genes - rank\n",
    "\n",
    "    # Sort genes based on total points\n",
    "    combined_ranking = sorted(points.items(), key=lambda x: x[1], reverse=True)\n",
    "    return combined_ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranked_list_rf_1 = [item[0] for item in ranked_list_rf]\n",
    "ranked_list_logreg_1 = [item[0] for item in ranked_list_logreg]\n",
    "ranked_list_svm_1 = [item[0] for item in ranked_list_svm]\n",
    "rankings = [ranked_list_rf_1, ranked_list_logreg_1, ranked_list_svm_1]\n",
    "final_ranking = combine_rankings(rankings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['S100B',\n",
       " 'NOL7',\n",
       " 'ILF2',\n",
       " 'HAT1',\n",
       " 'FBXO16',\n",
       " 'LYPLA1',\n",
       " 'TMEM14B',\n",
       " 'PDCL3',\n",
       " 'S100P',\n",
       " 'DKK3',\n",
       " 'PTPRC',\n",
       " 'GREM1',\n",
       " 'PTH2R',\n",
       " 'CSN3',\n",
       " 'SAA1',\n",
       " 'NOP10',\n",
       " 'TUBB6',\n",
       " 'PRNP',\n",
       " 'SEC13',\n",
       " 'NME2',\n",
       " 'ACTA2',\n",
       " 'NDP',\n",
       " 'DNAJC12',\n",
       " 'GPNMB',\n",
       " 'NPY1R',\n",
       " 'HLA-DPA1',\n",
       " 'BTG3',\n",
       " 'TFF3',\n",
       " 'SLC35B1',\n",
       " 'H2AFV']"
      ]
     },
     "execution_count": 402,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_rank = [item[0] for item in final_ranking]\n",
    "meta_rank[0:30]\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Since I have found the ranked list if features, I will now tune my hyperparameters on the test set that has been set aside "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tuning hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_feats_after_rfecv = ['S100B', 'NOL7', 'ILF2', 'HAT1', 'FBXO16', 'LYPLA1', 'TMEM14B', 'PDCL3', 'S100P', 'DKK3', 'PTPRC', 'GREM1', 'PTH2R', 'CSN3', 'SAA1', 'NOP10', 'TUBB6', 'PRNP', 'SEC13', 'NME2', 'ACTA2', 'NDP', 'DNAJC12', 'GPNMB', 'NPY1R', 'HLA-DPA1', 'BTG3', 'TFF3', 'SLC35B1', 'H2AFV']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#generating internal validation results \n",
    "\n",
    "auroc=[];  df= pd.DataFrame(); mcc_scorer = make_scorer(matthews_corrcoef);\n",
    "sampling_names = [\"RUS\", \"REDN\", \"CC\", \"IHT\", \"NearMiss\", \"EDN\", \"Tomek\", \"Allknn\",\"CondensedNN\", \"OSS\", \"SMOTE\", \"ROS\", \"ADASYN\", \"KmeansSmote\", \"BorderlineSmote\", \"SVMSmote\"]\n",
    "#sampling_names = [\"SMOTE\"]\n",
    "#sampling_techniuqes=[SMOTE(random_state=42)]\n",
    "sampling_techniuqes = [RandomUnderSampler(random_state=42), RepeatedEditedNearestNeighbours(), ClusterCentroids(random_state=42), InstanceHardnessThreshold(random_state=42), NearMiss(), EditedNearestNeighbours(), TomekLinks(), AllKNN(), CondensedNearestNeighbour(random_state=42), OneSidedSelection(random_state=42), SMOTE(random_state=42), RandomOverSampler(random_state=42), ADASYN(random_state=42), KMeansSMOTE(random_state=42),BorderlineSMOTE(random_state=42), SVMSMOTE(random_state=42)]\n",
    "classifier_names = [\"BRF\", \"EE\", \"Bagging\", \"RF\", \"ADA\", \"ET\",\"LOGREG\",\"DT\", \"LDA\", \"KNN\", \"NB\", \"SVM\"]; \n",
    "model =[]\n",
    "#classifier_names = ['RF']\n",
    "model_index = 0\n",
    "#feats = list(range(20,28)); \n",
    "feats = [14,15,16,17,18,19,20]; auroc=[]\n",
    "\n",
    "for n in range(0,len(feats)): \n",
    "    print(\"For \",feats[n],\" genes: \") \n",
    "    #subsetiing n+1 genes and sampling technique -> REDN \n",
    "    X_train_sel = train_data_feats[top_feats_after_rfecv[0:feats[n]]]\n",
    "    print(X_train_sel.shape)\n",
    "    scaler = [MinMaxScaler()]\n",
    "    scaler_names = [\"minmax\"]\n",
    "             \n",
    "    for s in range(0, len(scaler)):\n",
    "        print(\"Scaling scheme:\", scaler_names[s])\n",
    "        for samp in range(0,len(sampling_techniuqes)):\n",
    "            print(\"Scaling: \", scaler_names[s])\n",
    "            X_train_sel = pd.DataFrame(scaler[s].fit_transform(X_train_sel),columns = X_train_sel.columns)\n",
    "            #X_samp = X_train_sel; y_samp = train_labs\n",
    "            X_samp, y_samp=sampling_techniuqes[samp].fit_resample(X_train_sel,train_labs)\n",
    "            X_samp = pd.DataFrame(X_samp, columns = X_train_sel.columns)\n",
    "\n",
    "            for c in classifier_names:\n",
    "                name_id = sampling_names[samp]+ \"_\" + scaler_names[s]+ \"_\" + c + \"_\" + str(feats[n])\n",
    "\n",
    "                model_index = 0;  mcc_overall =[]; thres = []; auc_overall = []\n",
    "                if c == \"RF\":\n",
    "                    mod,score = model_tune_rf(X_samp, y_samp)\n",
    "                elif c == \"LOGREG\":\n",
    "                    mod,score = model_tune_logreg(X_samp, y_samp)\n",
    "                elif c == \"DT\":\n",
    "                    mod,score = model_tune_dt(X_samp, y_samp)\n",
    "                elif c == \"ET\":\n",
    "                    mod,score = model_tune_et(X_samp, y_samp)\n",
    "                elif c == \"ADA\":\n",
    "                    mod,score = model_tune_ada(X_samp, y_samp)\n",
    "                elif c == \"LDA\":\n",
    "                    mod,score = model_tune_lda(X_samp, y_samp)\n",
    "                elif c == \"KNN\":\n",
    "                    mod,score = model_tune_knn(X_samp, y_samp)\n",
    "                elif c == \"NB\":\n",
    "                    mod,score = model_tune_nb(X_samp, y_samp)\n",
    "                elif c == \"SVM\":\n",
    "                    mod,score = model_tune_svm(X_samp,y_samp)\n",
    "                elif c == \"BRF\":\n",
    "                    mod,score = model_tune_brf(X_samp,y_samp)\n",
    "                elif c == \"EE\":\n",
    "                    mod,score = model_tune_ee(X_samp, y_samp)\n",
    "                elif c == \"Bagging\":\n",
    "                    mod,score = model_tune_bagging(X_samp, y_samp)\n",
    "                \n",
    "\n",
    "                l=[score, c, feats[n], sampling_names[samp]]\n",
    "                row=pd.Series(l,['mcc_score','classifier','num_feats',\"sampling_names\"])\n",
    "                df=df.append([row],ignore_index=True)\n",
    "                print(df)\n",
    "                model.append(mod)\n",
    "                model_index+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model_index</th>\n",
       "      <th>mcc_score</th>\n",
       "      <th>classifier</th>\n",
       "      <th>num_feats</th>\n",
       "      <th>sampling_names</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>616</td>\n",
       "      <td>0.855828</td>\n",
       "      <td>ADA</td>\n",
       "      <td>17</td>\n",
       "      <td>IHT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>808</td>\n",
       "      <td>0.855813</td>\n",
       "      <td>ADA</td>\n",
       "      <td>18</td>\n",
       "      <td>IHT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>431</td>\n",
       "      <td>0.853170</td>\n",
       "      <td>SVM</td>\n",
       "      <td>16</td>\n",
       "      <td>IHT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>473</td>\n",
       "      <td>0.850131</td>\n",
       "      <td>ET</td>\n",
       "      <td>16</td>\n",
       "      <td>Allknn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>713</td>\n",
       "      <td>0.844225</td>\n",
       "      <td>ET</td>\n",
       "      <td>17</td>\n",
       "      <td>ROS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>425</td>\n",
       "      <td>0.839603</td>\n",
       "      <td>ET</td>\n",
       "      <td>16</td>\n",
       "      <td>IHT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>893</td>\n",
       "      <td>0.839275</td>\n",
       "      <td>ET</td>\n",
       "      <td>18</td>\n",
       "      <td>SMOTE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>617</td>\n",
       "      <td>0.838844</td>\n",
       "      <td>ET</td>\n",
       "      <td>17</td>\n",
       "      <td>IHT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>476</td>\n",
       "      <td>0.837515</td>\n",
       "      <td>LDA</td>\n",
       "      <td>16</td>\n",
       "      <td>Allknn</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>613</td>\n",
       "      <td>0.836369</td>\n",
       "      <td>EE</td>\n",
       "      <td>17</td>\n",
       "      <td>IHT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Model_index  mcc_score classifier  num_feats sampling_names\n",
       "0          616   0.855828        ADA         17            IHT\n",
       "1          808   0.855813        ADA         18            IHT\n",
       "2          431   0.853170        SVM         16            IHT\n",
       "3          473   0.850131         ET         16         Allknn\n",
       "4          713   0.844225         ET         17            ROS\n",
       "5          425   0.839603         ET         16            IHT\n",
       "6          893   0.839275         ET         18          SMOTE\n",
       "7          617   0.838844         ET         17            IHT\n",
       "8          476   0.837515        LDA         16         Allknn\n",
       "9          613   0.836369         EE         17            IHT"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_internal_validation = pd.read_csv('/home/shayantan/TNBC-single-cell/Reproducibility_best_results/consolidated_results_internal_validation.csv')\n",
    "df_internal_validation.columns = ['Model_index','mcc_score','classifier','num_feats','sampling_names']\n",
    "df_internal_validation[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 420,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('/home/shayantan/TNBC-single-cell/Reproducibility_best_results/model.pkl', 'wb') as file:\n",
    "    pickle.dump(model, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas import read_pickle\n",
    "import pickle\n",
    "\n",
    "\n",
    "model_gene_only = read_pickle('/home/shayantan/TNBC-single-cell/Reproducibility_best_results/model.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now I have found out the best classifiers from the training data using cross-validation. Time to test it using GSE Ids- 20271,20194,31519,106977"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "feats = [20]; sc = MinMaxScaler()\n",
    "X_train_sel = train_data_feats[top_feats_after_rfecv[0:feats[0]]]\n",
    "X_train_sel = pd.DataFrame(sc.fit_transform(X_train_sel),columns = X_train_sel.columns)\n",
    "\n",
    "val_data = pd.read_table('/home/shayantan/TNBC-single-cell/TNBC-archive/batch_corrected_validation_20271_20194_31519_dataset_combined.txt')\n",
    "val_labs_20271_20194_31519 =val_data['Labels'].astype('category').cat.codes\n",
    "val_data=val_data.drop('Labels', axis=1)\n",
    "val_data_feats_20271_20194_31519 = val_data[top_feats_after_rfecv[0:feats[0]]]\n",
    "val_data_feats_20271_20194_31519 = pd.DataFrame(sc.transform(val_data_feats_20271_20194_31519),columns = val_data_feats_20271_20194_31519.columns)\n",
    "\n",
    "\n",
    "val_data = pd.read_table('/home/shayantan/TNBC-single-cell/TNBC-archive/GSE106977/GSE106977_processed.txt')\n",
    "val_labs_106977 =val_data['Labels'].astype('category').cat.codes\n",
    "val_data=val_data.drop('Labels', axis=1)\n",
    "val_data_feats_106977 = val_data[top_feats_after_rfecv[0:feats[0]]]\n",
    "val_data_feats_106977 = pd.DataFrame(sc.transform(val_data_feats_106977),columns = val_data_feats_106977.columns)\n",
    "\n",
    "class_names = ['ExtraTreesClassifier']\n",
    "test_data = [val_data_feats_20271_20194_31519, val_data_feats_106977]\n",
    "test_labs = [val_labs_20271_20194_31519, val_labs_106977]\n",
    "test_names = ['20271_20194_31519','106977']\n",
    "\n",
    "classifiers = [model_gene_only[713]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold:  0.53\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      0.53      0.63        74\n",
      "           1       0.76      0.92      0.83       120\n",
      "\n",
      "    accuracy                           0.77       194\n",
      "   macro avg       0.78      0.72      0.73       194\n",
      "weighted avg       0.77      0.77      0.76       194\n",
      "\n",
      "Name:  ['ExtraTreesClassifier'] Sensitivity:  0.917 , Specificity:  0.527 AUPRC:  0.747 , AUROC:  0.782 , MCC:  0.496  , F1-weighted:  0.755\n",
      "Threshold:  0.531\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.51      0.61      0.55        46\n",
      "           1       0.72      0.63      0.67        73\n",
      "\n",
      "    accuracy                           0.62       119\n",
      "   macro avg       0.61      0.62      0.61       119\n",
      "weighted avg       0.64      0.62      0.63       119\n",
      "\n",
      "Name:  ['ExtraTreesClassifier'] Sensitivity:  0.63 , Specificity:  0.609 AUPRC:  0.68 , AUROC:  0.666 , MCC:  0.233  , F1-weighted:  0.626\n"
     ]
    }
   ],
   "source": [
    "df_results = pd.DataFrame();preds=[];probs=[]\n",
    "for i in range(0,2):\n",
    "    for j in range(0,1):\n",
    "        y_test_predictions, y_probs, sen, spe, auprc, acc, auc, mcc, f1, thresh = print_metrics_with_threshold_tuning(class_names, test_data[i], test_labs[i], top_feats_after_rfecv[0:17], classifiers[j])\n",
    "        name = class_names[j] + \"_\" + test_names[i] +\"_\"+ str(np.round(thresh,3))\n",
    "        preds.append(y_test_predictions);probs.append(y_probs)\n",
    "        l=[sen, spe, auprc, acc, auc, mcc, f1, name]\n",
    "        row=pd.Series(l,['Sensitivity', 'Specificity','AUPRC','Balanced_accuracy','AUROC', 'MCC', 'Weighted-f1', 'name'])\n",
    "        df_results=df_results.append([row],ignore_index=True)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnYAAAJfCAYAAADy95sHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAADUCUlEQVR4nOzdd3xV9eH/8de92ZMMAgkhgbA3yAZRpqioiNSBC1e1VXG3Vb/V1lrbWn9aa51Y68K9EHGwQRTZe4YQVggJIQnZ+97z++NCwiUh3ECSc+/N+/l49FHP55x784YLyZszPh+LYRgGIiIiIuLxrGYHEBEREZHGoWInIiIi4iVU7ERERES8hIqdiIiIiJdQsRMRERHxEr5mBzCb3W6nuLgYPz8/LBaL2XFERERETsswDCorKwkJCcFqrX1+rsUXu+LiYnbv3m12DBERERGXdevWjbCwsFrjLb7Y+fn5AY7fIH9//yb7Otu2baNPnz5N9v5ydvS5uB99Ju5Jn4v70Wfinpr6c6moqGD37t3V/eVULb7Ynbj86u/vT0BAQJN+raZ+fzk7+lzcjz4T96TPxf3oM3FPzfG5nO72MT08ISIiIuIlVOxEREREvISKnYiIiIiXULETERER8RIqdiIiIiJeQsVORERExEuo2ImIiIh4CRU7ERERES+hYiciIiLiJVTsRERERLyEip2IiIiIl1CxExEREfESKnYiIiIiXkLFTkRERMRLqNiJiIiIeAm3KXYVFRU899xz9OjRg5tvvrlBr920aRN33nknQ4YMoV+/flxxxRV88MEH2O32JkorIiIi4n58zQ4AsHfvXn73u9+xb98+DMNo0GtXrlzJnXfeSWxsLPfeey8REREsWLCAv/71r+zfv58nnniiiVKLiIiIuBfTz9jl5+czdepUbDYbX375ZYNeaxgGf/nLXwgMDOSjjz7i1ltvZcqUKbz22muMGzeODz74gF27djVRchERERH3Ynqxq6ysZPLkyXz22Wd06tSpQa/dtm0b+/bt49JLL6VNmzZO+26++WYMw+Cbb75pzLgiIiIibsv0S7GtW7fm6aefPqvXbt68GYB+/frV2te/f3+nY0RERKTxVNns5JSUmx3Drfj7mH6+zPxidy7S0tIAiIuLq7UvJCSE8PDw6mNERESkcczZlsZtH68gv6zS7Chuwcdi5zcDD3BeQgmZBaEkFnYiJizSlCzmV8tzUFxcDEBQUFCd+4OCgigqKmrOSCIiIl7vD3PXq9Sd5Nr4bTzr8/+ILT9Eu1ZFfLZhoWlZPPqM3ZkYhoHFYnHp2G3btjVxGli/fn2Tfw1pOH0u7kefiXvS5+J+zPpM0o7ppMkJI0N2M7PNy4TYyplw9HMWtrmWY5Xxpn02Hl3swsLCgJozd6cqKSmpPuZM+vTpQ0BAQKNlO9X69esZNGhQk72/nB19Lu5Hn4l70ufifsz8TCyfJQO26u2Y0AAsuHYixZuMCNrOrPb/jxCL415Du8WHKosfwzq3Z1DXpvlsysvL6z0Z5dHFLiEhAYCMjIxa+/Lz8ykqKqJ3797NHUtERLxIeZWN336+itlb0yivsp35Bc3EMAwsn5ozpVeFzXkBgH1PTCXIz6MrRcNlLoIfnwObo9SVWkOY3+Y68vxjaOPi1cKm4NGfwsCBAwHHyhPXXXed075169YBMHjw4GbPJSIi3uPjDft5f91es2OcRsMm9ZdGkv49/DQV7I5SV+nfmnnRU8j3izY5mIc9PJGamur0lGuPHj3o1asX8+bNczprZxgG7777Lr6+vkyZMsWEpCIi4i1ScwrMjuDW2oYFEujrY3aM5nN4Pvw0pbrUEZzAwfNec4tSB25wxm7Pnj3s2bPHaSw3N5d58+ZVb48ePZqgoCAmTZpEUlKS074///nPTJ8+nRtvvJFbbrmF8PBwvv32W9asWcMDDzxAYmJis/1aREREWpK48CBenjrU5QcVvUJEbwhOgKK9ENIRxi+lsjADcI+Hi0wvdj/88AOvvPKK09iePXt44IEHqrcXL15M+/bt63z9gAED+Pjjj/nPf/7Dyy+/TGVlJZ07d+af//ynztaJiHiwnOJy7vtqDRsO5Zh6wTG72HkS3j9N7Mdj4/uYlKbGhg0bqm9JMou/j7VllTqA4PYwfgms/jUMextCEqCw9r3+ZjG92N13333cd999Lh2bnJxc53jv3r2ZOXNmY8YSERGTPfrtej7dtN/sGLX4Wi0EuMGlR38fq1vkaJFCOsA48+aqq49H3WMnIiItx7aMPLMj1Ck2vO5J8cVLpf4PDs8783FuQsVORETERcM7tObqfh3MjiHNZferjkuuy6c4pjfxAKZfihUREfeScrSAvy/ayqG8kuqxgsICwtfkNmuOXVnOT6N+Ov1C+rczZ/1NgCA/H+JbBbe8e8paql0vwoaHHf9tL4dtf4W248HNP38VOxERcTLl7aW1ShUAR0pqjzWjxMgQusaEm5pBWojtz8Lmx2u2o4fDhXPcvtSBLsWKiMhJ8ksr6i51biAq2N/sCNISbH3audTFjIJxC8A/os7DyyqLOJRrzgogddEZOxERqWY33HMlgyl9E+gc7dra3yJnxTBgy5Ow/W81Y23Hwui54BtSx+F2dh9Zy/r986ioKnXa5+vj19RpT0vFTkSkBSqpqOKVn3eRfMrZuVPXQg3x92X2bWPYnbKbbl27NWPCGjGhgfSJjdC9bdJ0DAM2/QF2Pl8zFjsRLpwNvsG1Ds8tOszK1K85Wniw1j4LFjpEmzfPoYqdiEgLdPcXq/lg/ZnXP/XzsTK+WxwRhYcZ1C2uGZKJNDPDgA0PQfJLNWPtLoMLvgCfQKdDK6vK2XhwITsP/4KBvdZbRQS3JaqqF23CzXtyWsVORKQFWrTbtZnyQ/31Y0K8nMXiWE3ihPZT4PxPwafmnk7DMDiQs401e+dSUlH7HlRfqx/9EyfQu90oNm7c1PSZ66G/sSIiLVCVvfbZhrrcMaxLEycRcQM9fwe2csjbAiM/AGvNPXIFpTms3juH9GO763xpQlQvhnW6gtBA86biOZmKnYiIB9iWcYzvd6ZTWmk788EuKK6octp+8crBhAY43/DdvU04IzvGNMrXE3F7ff4Ihh0sjglDbPYqth36kS2HlmKzV9U6PCQgguGdJpMQ3au5k9ZLxU5ExM1tz8xj+Es/NFqpq8v1A5OICQ0884Eins5e6XhIovuD4HvK8nDHS11G3h5Wps6hoPRorZdbLFZ6x19A/4Tx+Pm43xQ8KnYiIm5uXiOeqTudAF9NayotgK0CVkyDQ7Mh60e48GunByRKKgpZt+879h7dVOfL24YnMbzzFCJD2jZP3rOgYici4ubKqpq21F3ULY7wQPc78yDSqGxl8NPVcPg7x3bGfNj3AXT5NXbDTnLGajYcmE+lrazWSwN8QxiSNInObQa6/bQ7KnYiIm4ir7SCRbszKCirdBpff8h5jdYLOrVhbJfYRvma7SOCuW5Ax0Z5LxG3VVUKy6dA5oKasZ6/g853kF10iJV7vian6FCdL+0WO5RBHS4hwK/2fHbuSMVORMQNFJZVMvzf35OSXXjGYy/o1IY/X9y/GVKJeIGqYvjxCjiytGas9x+p6PVHNuz9huSMVRjUXnElMiSOEZ2nmDon3dlQsRMRcQPL9x5xqdQB+Fl1P5yISyoLYdllcPSn6iGj71Psi72StRv+RWll7b9zvj7+nJd4ET3bjcRq8WnOtI1CxU5ExA0UldeeTqEuFguM66oVIETOqCIPll4KOauqhzI63MHaqkhykz+p8yUdovsytNPlhAS0aqaQjU/FTkTEDXWMCmFMZ+f76AJ8fZjcpz2jOrUxKZWIhyjPxVh6MZbcddVDayLGscNoDcWHax0eFhjFsE5X0j6qe3OmbBIqdiIibmhIQmv+N22k2TFEPEpFVRnpx3aTeeQXehXu48R5t1WRF7ErbGCt460WH/q0H02/9mPx9fGrtd8TqdiJiIiIxyouzyMtdycHc3aQmb8Xu+GYHuhgzHVckvUJ28KHkBI6oNbr4iO7MzTpcloFe9fqKip2IiIi4jEMwyC3OIO03B2k5ewkpzi9zuNKfcP4Ju42bBZH1bFafGkX0YXE6F60j+pJsH9Yc8ZuNip2IiJNoKi8koyCUpePzyx0/ViRlsZmr+JIwT4O5uwgLXcnxeV5TvuDqwqIrDxKelBnp3Ffv1YkRfUgIaon7SK7ueUSYI1NxU5EpJG9+vMuHp6zjip77bmxRLxNlb2S3KIM7IZrT3Y3RElFAWk5Ozl0LLnOFSEAQqvyuPjIJwTbClkaM5WCyGEkRPciMaoXMeGJWC0ta3ogFTsRkUZUZbPzxA+bVOrE69kNO7szV7N+f93LcDWHsMpjXJz1CaG2AgDGH5uH5cKZEBBlSh53oGInItKISitttZYEOxv92kU2QhqRpnGmZbiamo/Vjy6BwQzZ+za+x0sd1gAsoz5r0aUOVOxERM6oospWx4JDdSuvstUa69La9Zu0fa0WRneO5eHRvVx+jUhzqagqY8OB+addhqspBfqFkBDVk4SoXrSjHN8fL4WKo46dPkEw+huIndCsmdyRip2IyGmkZhdy7Xs/sunwsbN+j9AAX5Ifn9J4oURMYBgG+7I3s3bvd6ddhis6pF2jf12rxYfo0PYkRveidViC4365Y5thyUQozz7+xUNg9LfQdkyjf31PpGInInIaf1mw+ZxKnYg3yC89yqrUOWTk7alzf7Muw5WzDpZOhIrjfy99w2DsDxBzftN/bQ+hYicichoHcovP+T26RHvnXFni/arslWxNW8bWQ8uqJ/09WVhgFMM6X0n7yGZahit7FSy9GCqP31Pn1wrGzofWw5rn63sIFTsRERf5Wi1YLRaXj+8WE85rV+uHjnie9GO7WZU6h8KynFr7rBYf+rYfTd/mXobr2OaaUucfBeMWQlTtZcJaOhU7EWnRSiuruPfLNSzanUGlze60L7ek3Gl70d0XcUGnts0ZT6RZFZfns3bft+zP3lrn/rhWXRje+UpzluHq+huwlcD2f8C4RRDZr/kzeAAVOxFp0V79OZn31qaaHUPEVHbDxq7DK9l4cCGVtvJa+wP9QhmadDlJMf2xNOCsdaPr8RAkTYeAaPMyuDkVOxFp0XZl5bt8bPtWwU2YRMQcRwsPsnLPbHKLM+rYa6FH3HDO6zCRAN+g5g2WtRwiB4JfqPO4Sl29VOxERM7Az8fK/Rf0IEkPQogXKa8qYf3++ezOXAN1zEkXHRrPiM5X0TqsffOHS/saVlwLrc+HMd+Br/5R5SoVOxGRkzw/eRA3DExyGgsL8CPYX98uxTsYhkFq1gbW7f+essraT377+QQwsMMldI8bZs46qwc/hxU3gFEFWctg3X0w/H/Nn8ND6TuViMhJwgP9aBvWzJecRJpJXkkWq1K/JjN/b537k2L6MyTpMoL9w5s52XH7P4KVN4Nx/EGmsK7Q9ylzsngoFTsREREvV1ZZzLb05exI/7nOOenCg1ozvPOVtIvoakK64/a+C6tup/qycHgPGLcYght/RQtvpmInIiLipcoqi9ie/jM7M36hylZRa7/V4kv/hLH0aT8aH6uJlWDPm7DmNzXbrfo4pjQJ0vRCDaViJyIi4mVKKgrZnr6c5IxVVNkr6zwmPrIbwzpdSXiQyU+ZJr8C6++r2Y4cAGMXQmBr0yJ5MhU7ERERL1Fcns+29OXszlyNzV5V5zHB/uEM7XQ5HaL7mjsnHcDOf8HGR2q2owY7lgkLiDIvk4dTsRMREfFwRWV5bEtfxu7MtXXeQweOSYb7xF9I97hh+PkENHPCOtjKYN+7NdutR8CYH8C/lWmRvIGKnYiIiIcqLMtl66Fl7Dmy/rSFLsg/jD7xo+keOxRfH/9mTlgPn0DHfXSLxkBAa8d8dX6aK/JcqdiJiIh4mILSbA5VrGXbui8xsNd5TLB/K/q2H0PX2MH4Wv2aOaGLAtvA+CWOQucbYnYar6BiJyIi4iHySrLYkraUfUc3YdSxWgRASEAE/dqPpUvbQeY+6Xoqw4BjmyDqPOfxoFhT4ngrN/rERUREpC7Hio+wJW0J+7K3UNfyXwBhgVH0az+Wzm0GYrX6NG/AMzHssP4BSHkdRn0GCVPNTuS1VOxERETcVG7RYTanLeFAzrbTHhMe1Jr+CeNIiumP1eJmhQ4cpW7NbyH1v47tFdNgwk/Qepi5ubyUip2ItAh7cwpZuDuDskrnG8y3Z+aZE0ikHtlFh9h8cAlpuTtOe0yAJYxh3S6jY+t+5qzp6gq7DVbfAfveqxlL+BVEDTQvk5dTsRMRr1ZQVsFf5m/h5Z93YbPXfQlLxF1kFRxkS9piDh1LPu0xkcGx9E8cR/b+CjrFDGi+cA1lr4KVt8CBj2rGOt4Mw98Bd7tU7EVU7ETEKxmGwccb9/P7b9aTWVjq8uv8ffQDR5rfkfz9bE5bzOG8lNMeEx0ST7/EcSRG9cRisZJzYH0zJmwgeyX8ciMc/LxmrPMdMGSmSl0TU7ETEa+zPTOP+75aw4+pRxr0utAAX0Z31tqU0jwMwyAzfy+b0xaTmb/3tMe1Dk2gf+J42kd2N3+lCFfYymHFdXBoTs1Y17th8CvgrpeMvYiKnYh4lCqbndUHs8kqKqtz/4p9Wbz80y6q6rjsmhgZwpV9EqjrR2NogC/X9O9IYqTm0pKmZRgGGfl72HxwMUcK9p/2uDZhHeifOJ52EV09o9CBYzWJn34Fh7+vGev+AAx8ETzl1+DhVOxExKP86t0f+XbHoQa9xt/Hyu/H9uax8X0I9te3PTGHYRikH9vN5rTFHC08eNrj2oYnMSBxPLGtOntOoTshc4lzqev5BxjwrEpdM9J3OBHxGAdyixpc6i7u0Y6Xpgyha0x4E6USqZ9hGKTl7mRz2hJyik7/5zcuogv9E8YR26pTM6ZrZPGTYMhrsPYe6P0E9Htapa6ZqdiJiMfIL6t0+djEyBBevHKw49KrfrCICQzDzsGcHWxOW0xuccZpj4uP7Eb/hPG0Ce/QjOmaUNe7IXIQtB5qdpIWScVORDxWWIAf47o6L0fkY7UwKqkNdw7vqsuuYgq7YedA9lY2py0hr+T0D/AkRPWkX8I4YsISmjFdI6vIAyzg38p5XKXONPquJyIeq2NUCF/dNsbsGCIA2A0b+45uYUvaEvJLj572uMTo3vRPGEd0aHwzpmsC5TmwZCL4BMDY+eAXZnYiQcVORETknNjtNlKPbmRL2lIKy3JOc5SFjq370j9hHJEhXrDofVkWLLkI8rY4tn+8AsYt1hx1bkDFTkRE5CzY7FWkZm1gS9oyispz6zzGgoWkmP70SxhLRLCXzJFYmglLxkP+ieXOLJB0s0qdm1CxExERaQCbvYqUI+vYemgZxeV5dR5jwUqnNgPolzCWVkExzRuwKZWkw+JxULjbsW2xwvB3HcVO3IKKnYiIiAuqbJXsPrKGbYd+pKSioM5jLBYrXdoMol/CGMICo5s5YRMrPugodUWpjm2LD4z4ADpOMzeXOFGxExERqUelrYLdmavZeuhHyiqL6jzGavGha9vB9G0/htDAyGZO2AyK9jpKXfEBx7bFF87/BBJ/ZW4uqUXFTkREpA6VVeXsylzJ9vSfKKssrvMYq8WXbrFD6Nt+NCEBEc0bsLkUpMCScVByfHJlqz+M+gLaX2FuLqmTip2IiMhJKqrK2JnxCzvSf6a8qqTOY3ysfnSPHUaf9hcS7O/lq5pseLim1PkEwgWzod0l5maS01KxExG3szD5MA/MXsuBY85nSeyGYVIiaQlKygtIzlzNzsMrqLCV1XmMr9WfHnEj6B1/AUH+oc2c0CQj3oPFY6EwBUbPhdjxZieSeqjYiYhbMQyDOz5dSXp+3WdKRBqTYdjJyE8lOWMVB3N2YmCv8zg/nwB6xo2kV/woAv1CmjmlyQKiYNwiKNwDMSPMTiNnoGInIm6l3Ga4XOo6RLaQMybS6MorS9iTtZ7kjNUUlGWf9jg/n0B6tTufXu3OJ8AvuBkTmqgiD/wjnMcCYxz/E7enYiciHql7TDh/mzTA7BjiQQzD4GhhGsmZq9h3dAt2o+q0xwb4BtOr3fn0bHc+/r6BzZjSZEdXwo+XweBXoeP1ZqeRs6BiJyJuLcDXSs4z1zmNWbAQ6KdZ7sU1lbZy9h7dRHLGKnKLM+o9NiK4Dd1jh9OlzSD8fAOaKaGbyFoOyy6DqiJYebPjQYmEq8xOJQ2kYicibs2ChSA/fauShjtWnEly5ipSszZSaSs/7XFWiw8dWvehe+ww2oYnYbFYmjGlm8hc4ljv1Xb8Ngj/KAjrYm4mOSv6bikiIl7DZq9if/ZWkjNXk1Wwv95jQwMi6RY7jK5tB7ecJ1zrcng+/DQFTjwJHBgL4xdDq16mxpKzo2InIiIer7Ash+SMNaQcWUd5Vd2TCYPjDHD7qB50jx1Ou8iuWC3WZkzphtK/hZ9+BfYKx3ZQPIxfAuHdzM0lZ03FTkREPJLdsHEodxe7MlZzOG93vccG+oXSLXYI3doOIzQwonkCuru02bDiOrBXOraDEx2lLqyzubnknKjYiYiIRympKCAlcy3JmWsoqciv99jYVp3oETechKhe+Fj1I6/agc/glxvAsDm2Qzs5Sl1IB3NzyTnTn3IREXF7hmHUTCScuwPDqHsiYQB/n0A6tx1E99hhRAS3acaUHqL4IKy8qabUhXV1lLrg9ubmkkahYiciIm6reiLhzNUUlJ5+ImGA1qHt6R43nKTW/fD18W+mhB4oJBGGvgmrboPwno4HJYLizE4ljUTFTkRMUV5l47UVyezIdL6UduToUZMSibswDIPsojR2Zaxif/YWbPbTTyTsY/WjU0x/uscNp3Wozji5rNOt4BMMbcdAoM5qehMVOxExxQOz1/LfVSlmx5AmZhh2yipdWyLOMOykHdtJcsZqcosP13tsq6A29IgbRqc2AwnwDWqMqN7NVg4+p0y43OFac7JIk1KxExFTLNxd/w/uE8IC9W3KE9kNGzsP/8LWQz9SVlnUKO9ptfiQGN2bHnHDW+5Ewmdj5/Nw4BMYt6j2GrDidfQdU0RMUWkzXDrutiGa/d7TZBUcYOWe2RwryWyU9wsJiKB79UTCYY3yni3Gtr/Blicc/730Ehi3APzCzc0kTUrFTkSa1MLkw6w5mI39lB5XUFbptP3sZQOJDPbnwIEDdOjgmHKha0w4F3bS/T+eoryyhPX757H7yJpGeDcL7SO70z1uOPGR3TSRcEMZBmx9CrY9XTPmEwDo99HbqdiJSJN5e/Ue7vxspUvHXj+wI+0jQljvV8CgQV2bOJk0JsMw2JO1nnX7fqhz1QerxRd/34A6Xlmbv28wHaP70C12KKGBkY0dtWUwDNj8OOz4Z81Y2/Eweg74hpiXS5qFip2INJkvthxw+dhAX58mTCJN5VjxEValzubIadZl7RRzHkOSJukSanMxDNjwMCT/u2Ys7hK44CvQQyYtgoqdiDSZ8kqbS8dd2KkNrUMDmziNNKZKWwWbDy5m++Gf6pwsODwohhGdryQuQvdINhvDDuvug5TXasbiJ8Ooz2o/ESteyy2KXUFBAS+//DKLFy8mKyuLiIgIRo8ezYMPPkhMTMwZX7948WLeffddUlNTKS0tpX379owfP57bbruNVq1aNcOvQERccdOgTnSMcr4U1K5VMNMGdDQnkJyVgzk7WL33G4rL82rt87H60i9hHH3iL9QSXs3JsMOa30DqWzVjCb+CkR+BJmtuUUz/W1dSUsJNN91EamoqN954I3369GH//v28/fbbrFq1ii+++ILIyNPfZ/Hiiy/yxhtv0LdvX+69916CgoLYtGkT//3vf/nuu++YPXs2oaGhzfgrEpHTuXVoZ8Z2iTU7hpylorJjrN77DWm5O+vc3z6yO8M6TyYsMLqZkwk7/ulc6jpcDyPeB5XrFsf0T3zWrFkkJyfz5z//mRtuuKF6vGfPnsyYMYOZM2fy2GOP1fnaY8eO8dZbbxEfH8+HH35IQIDjVPPUqVOJiIhg5syZfPHFF9x6663N8UsREfFKdruNo5W72Lnha6rslbX2B/u3YlinK0iM7q255czS9R5I+wpy10HSdBj2Nlh132pLZPpzz3PnziU4OJirr77aaXzChAnExsYyd+5cDKPu+a4yMzOpqqqib9++1aXuhEGDBgFw+LBrk6CKiEhtmfl7+WbTS2RWba1V6ixY6d1uFFcNfJgOrfuo1JnJvxWMnQ/9noHh76jUtWCmnrErLi4mJSWFQYMG4e/vfA+AxWKhf//+zJ8/n0OHDpGQkFDr9QkJCfj7+7N///5a+w4dOgRA586dmyS7iIg3K6ssYt2+H9iTtb7O/TFhiYzochVRIVo83hR1nfAIiII+f2z+LOJWTD1jl5aWBkBcXN3fGGJjY52OO1VoaCi//e1v2bVrF3/5y19ITU3lyJEjLFiwgNdee41u3bpx5ZVXNk14EREvZBh2dmeu4av1L9RZ6gJ8gxnZ5VdM6vdblTqzVJXCj5cTnT/H7CTihkw/YwcQFFT33DonxouKTr/O4L333ktUVBR///vf+eijj6rHx44dy7PPPktgoKZQEBFxRW7RYVamfs3RwoN17u/SZhCDkyYR6KdJbk1TVQI/ToYji+nAD7CvGyTdZHYqcSOmPzzhivru2/jggw/4+9//zoUXXsgVV1xBUFAQmzdv5v333+euu+7iv//9r0tTnmzbtq0xI9dp/fq6L2mIufS5NJ3CwkKn7d27dxOen37G1+kzaV42o5IjldvJse0Bal/iC7CEE+83iKDC1mzfsqv5AwoAVnsJXQ49SFjpBgAsGBzevZyM3J4mJ5NTmfk9zNRiFxbmmIn8xJm7U50YP910Jampqfz973/n/PPP54033qgeHzduHD179uSBBx7g9ddfP+1TtSfr06dPrQcwGtP69eurH+gQ96HPpWmFrc6BrJLq7W7dujHoDNOd6DNpPoZhcCBnG2v2zqfEVlBrv6/VjwGJE+jVbhQbN27S52KminxYNgmOlzqA9NZ3Ez/xNdqZGEtqa+rvYeXl5fWejDK12CUkJGCxWMjIyKhzf3q641/2JxYEP9XKlSux2WyMHz++1r6xY8disVhYs6YxFqMWEfEuhWU5rEr9hvRjyXXuT4zqxdBOkwkNjGjeYFJbxTFYegnknPTzbMBzZJaOI968VOKmTC12QUFB9OzZk507d1JWVuZ0P5zNZmPTpk3Ex8fTrl3d/x4pKysDHO31VOXl5RiGQWVl7TmXRERaKpu9im2HfmTLoaXY7FW19ocERDC802QSonuZkE5qKc+BJRfBsY01YwP/DT0eAN2yIHUwfR67q666irKyMj755BOn8Tlz5pCbm8vUqVOrx1JTU52ekB0wYAAAP/zwQ6257hYuXOh0jIhIS5eRt4c5G19i48GFtUqdxWKlb/sxTBn4sEqduyjLgsVjnUvdkNcdpU7kNEx/eGLatGl8++23PPfcc6Snp9O3b19SUlJ455136NGjB7fffnv1sZMmTSIpKYl58+YBMHjwYCZOnMiCBQu4/vrrueyyywgNDWX79u18+umnREdHc/fdd5v1SxNpUaps9lq33dtPM7m4NK/SikLW7vuOvUc31bm/bXgSI7pMISK4bfMGk9OzVznO1OVtPT5ggWFvQefb632ZiOnFzt/fn7fffptXX32VefPm8fHHHxMdHc20adO4//77CQ4Orvf1L774Ih999BFff/01L7zwAlVVVbRp04arrrqKe+65p3ouPBFpGnmlFUx7fzmLUzJV5NyM3bCzO3M16/fPp9JWVmt/gG8IQ5Im0bnNQK0a4W6svtDrcVh5o2N7+Hua1kRcYnqxA8dTr48++iiPPvpovcclJ9e+ydfX15fp06czffr0poonIvV4fUUyC3fX/QCUmCe76BCr9nxNdtGhOvd3ix3KoA6XEOBX/z+exUQdp4FRCVZ/6HCd2WnEQ7hFsRMRz7X/2OknED9V5+iwJkwiABVVZWw8sIBdGSsx6piTLjIkjhGdr6JNeKIJ6aTBkm42O4F4GBU7EWlUVosF6ylX9aJDAvjdmN4kRmrFgqZiGAb7srewdu+3lFYW1trv6+PPeYkT6dluBFaLFoh3OwUpsP4BGDkLAqLNTiMeTMVORBrVq78ayl0jupkdo0UpKM1mZerXZOTtqXN/h+i+DO10OSEBZ16FR0yQvxMWj4OyTMcDE+MXg3+k2anEQ6nYiYh4qCp7JVvTlrH10DLshq3W/rDAKIZ1vpL2kd2bP5y4Jm8rLB4P5Ucd2wXJkL8LYkaYm0s8loqdiIgHSj+2m1Wpcygsy6m1z2rxoW/70fRtPxZfHz8T0olLcjfC0osckxAD+IbCmO9V6uScqNiJiHiQ4vJ81u77jv3ZW+rcH9eqC8M7X0mr4JhmTiYNkrMWlkyEyjzHtl84jJmnUifnTMVORFzy5srdfLpxP6WVzpf89ubWvlFfGp/dsLHr8Eo2HlxIpa32MoqBfqEMTbqcpJj+mpPO3R39BZZdCpUFjm2/CBi3AKKHmBpLvIOKnYic0bI9mdz9xWqzY7RYRwsPsnLPbHKL65ov0EKPuOGc12EiAb5BzZ5NGihrOSybBFXFju2AaBi3CCIHmBpLvIeKnYic0fq02vdxnU5MaGATJmlZyqtKWL9/Prsz10Adc9JFh8YzovNVtA5r3/zhpOGyV8PSS8BW6tgObAPjFkNEH3NziVdRsRORM3J1obChidFc1C2uSbO0BIZhkJq1gXX7v6essrjWfj+fAAZ2uITuccOwWqwmJJSz0qo3RA2EoysgKA7GLYFWPcxOJV5GxU5EGuz68zoy4wLnH0ih/r70ahuB9dTZiaVB8kqyWJX6NZn5e+vcnxTTnyFJlxHsH97MyeSc+R1/6nXVbdD/WQjvanYi8UIqdiLSYO1aBTO8g566bExVtgo2py1he/pPdc5JFx7UmuGdr6RdhMqAR/MLhwu+NDuFeDEVOxERk6Xl7mR16jcUlR+rtc9q8aV/wlj6tB+Nj1Xfsj3K/k+gIhe63WN2EmlB9F1CRMQkRWV5rNn7DQdzd9S5Pz6yG8M6XUl4kNYO9Th734fVt4FhB6sfdLnT7ETSQqjYiYg0M7vdxo7DK9h0cBFV9opa+4P9wxna6XI6RPfVnHSeKPV/sPpOqh87Sv4PJN0CPv6mxpKWQcVORKQZHSnYz8o9s8krOVJrnwULPduNZEDiRfj7atoYj5TyOqw96dJrRD/HPHUqddJMVOxERJpBWWUx6/f/QMqRdXXubx2WwIjOVxEd2q6Zk0mj2fUSbHiwZjtyoGNFiQBdSpfmo2InItKEDMNOypH1rN//A+VVJbX2+/sEMijpUrq1HYJFc9J5rh3PwaZHa7ajh8HYeeAfYVokaZlU7EREmsix4kxWps4mq+BAnfs7txnI4I6TCPIPbeZk0qi2/hW2/qlmO2YUjPnOMbWJSDNTsRMRaWSVtnI2HVzMjvSfMbDX2t8qqA0jukwhtlUnE9JJo9r6F9j6VM1227Fw4TeOyYhFTKBiJyLSSAzD4GDOdtbsm0txeX6t/T5WP/onjKd3/CjNSectIvqDxReMKoi9CC78GnyDzU4lLZi+s4iINILCslxWp37DoWO76tzfPrIHwzpPJiwwqpmTSZNKmALnfwT7PoBRn4KPnmYWc6nYiYicA5u9iu3pP7E5bQk2e2Wt/SEBrRjaaTKJUb00J523SrwGEq4Gfb7iBlTsRETOUkZeKqtS55BfmlVrn8VipXe7UfRPHI+fT4AJ6aTR2W2w7Wnoeg8EtXXep1InbkLFTkSkgUorili37ztSj26sc3+b8I6M6DyFyJDYZk4mTcZeBatug/0fQNpXMH4pBLY2O5VILSp2IiIuMgw7uzPXsn7/D1TYymrtD/ANZnDHS+nSdpDmpPMm9kr45WY4+KljO38bpLwGff9U/+tETKBiJyLVDMNgf24ReaXO94ql59eeWLelySvJ4ueUz8kuTKtzf9e2gxnU8VIC/UKaOZk0KVsFrJgGh2bXjHW5C/o8YV4mkXqo2IkIAFU2O1PfXcZ3O9LNjuJ28kqy+GHLTMqrimvtiwyOZXiXKbQN79j8waRp2crgp2vg8Lc1Y91mwKD/6J46cVsqdiICwJqD2S6Xupb0I62oLI8F2/5Xq9T5Wv0ZkDiBXu3Ox2r1MSmdNJmqUvjpKsiYXzPW42E473mVOnFrKnYiAsDR4nKXjx0Q3zLmYiutKGLB9rcoqXCebDghqhfDO08mJCDCnGDStKqK4cfJcGRJzVivx6H/31TqxO2p2Im0QBVVNgrLq5zGCsqc76sLD/SjU5TzskgBvj5c3rs9087r2NQRTVdRVcbC7W9TUJrtNN617RBGdpmqOem8VWUhLLsMjv5UM9b3KejzJ5U68QgqdiItzP9bsp2/LNhMaaWt3uNGd27L17ePbaZU7qXKVsHiHe+SW3zYabxj676M6HKVSp03s/iC1a9mu//foffj5uURaSAVO5EW5FhJOY9/vwHDMDuJ+7LbbSzb9SFHCvY7jbeL6MoF3a7DqmlMvJtvEIz+BpZeCu2nQM+HzU4k0iAqdiItSEZBqculrkebVk0bxg0Zhp2fUj7j0LFkp/GYsETG9rwZH6u+ZbYIviEwfgno8xYPpD+1Ii2Y1WIhIsjPaczHamFUUlseG9/HpFTmMAyDVanfsO/oZqfxyOBYJvS+FT8ff5OSSZMqy4LD30OnW53HVerEQ+lPrkgL1r1NONv+MNnsGG5h48EFJGeuchoLC4zmoj53EOAbbFIqaVKlGbB4PBTsdDw00f0+sxOJnDPdLCIiLd62Q8vZkrbUaSzYP5yJfe4g2D/MpFTSpEoOwaLRjlIHsOEhKNxjbiaRRqAzdiLSoqVkrmXd/u+dxgJ8g7mo9x2EBbaM+fpanKL9sHgcFO9zbFt8YORHENbF1FgijUHFTkRarP3ZW/llz1dOY74+/kzofRuRIW1NSiVNqjDVUepKDjq2rX5w/meQMMXUWCKNRcVORFqkw8dSWJ78CQY1jwlbLT6M7zmdmLAEE5NJkylIdtxTV3p86TxrAFzwJcRfZm4ukUakYiciLU5WwUGW7JyF3aiZpNmChdE9biAuQpfjvFL+DseZurIjjm2fQLhwDsRNNDeXSCNTsRORFuVYcSaLdrxDlb3CaXxk11/RIbq3SamkSR3bAksmQPlRx7ZPMIyeC7HjzM0l0gRU7ESkxSgsy2HBtv9RUVXqND4k6XK6th1sUippcqWHoTLP8d++oTDme2hzgamRRJqKip2IF0o7VszfF29lf26x03hxeaVJicxXUl7A/K3/o7Sy0Gm8f8J4esePMimVNIt2l8Coz2H1rx1n6loPNzuRSJNRsRPxQte+/yNrDuaYHcNtlFeWsGD7/ygqz3Ua7xE3ggGJE0xKJc2q/ZXQdhz4aV5C8W6aoFjEy1TZ7C6Xusgg718mq9JWzsId75BXcsRpvFPMeQzrdAUWi8WkZNJksn52TEB8KpU6aQFU7ES8jHHmQwCwWOD+C3s2aRaz2exVLNk5i+zCNKfxhKiejOp6NRaLvgV6ncxFsHTi8WlNMsxOI9LsdClWxENVVNmYuXI32zLznMbtdufjrBYLc389ttbr+8ZFEt/Ke9dAtRs2lid/TEae8zJRbcOTGN39BqxWH5OSSZNJ/x5+mgr2cijcDSunw7iFZqcSaVYqdiIe6uE563j9l91nPM5qgUt6xDdDIvdhGAa/pHzFgZztTuPRIfGM73ULvj5+JiWTJnNoDvx8DdiPPyAUnABDXjc3k4gJVOxEPNTC3a5dZgoNaFklxjAM1u37jj1Z653GWwXFcFGf2/D3DTQpmTSZg1/AiuvBqHJsh3SE8UshtKOZqURMoRtMRDxUpc1+5oOAW4d0buIk7mXroWVsP/yz01hIQAQT+9xBoF+oSamkyez/CFZMqyl1oV1gwnKVOmmxdMZOxEs8c+kAokMCnMY6R4cxtkusSYma366MVWw4MN9pLNAvhIl97iAkIMKcUNJ09r4Hq26j+pGh8O4wbgkEtzM1loiZVOxEvMT1A5PoGNVyz0jtzdrEqtQ5TmN+PgFc1Pt2WgXFmJRKmsye/8Ka31Bd6lr1hnGLIaitqbFEzKZLsSLi8dJyd/FTymecPNmLj9WXCb1uJTq0ZT040iIYBhz6murPO6K/4546lToRFTsR8WyZ+XtZtusDDKPmnkOLxcrYHjfRtlWSicmkyVgsMOoLiJ0AUYNg/BII1FlZEdClWBHxYDlF6Sze8R42e9VJoxYu6HYt7aN6mJZLmoFvEFw4B+wV4B9hdhoRt6EzdiLikfJLjrJw+9tU2sqdxod3vpJOMQPMCSVNwzAga3ntcd9glTqRU6jYiYjHKSrLY8H2tyirLHYaH9jhYnrEDTcplTQJw4AtT8Ci0bDjObPTiLg9FTsR8SilFUUs2P4WxeX5TuO94y+kb/sx5oSSpmEYsPH3sP3vju1Nj8LBL83NJOLmdI+diJvLL61gS8Yx7IbzeFmVzZxAJqqoKmPh9rcpKM12Gu/adjCDO16KxWIxKZk0OsOA9Q/A7pdrxtpdBvGXmZdJxAM0qNjl5uYyc+ZMli9fTmZmJv/9738ZPHgwAG+//TaXX345bdq0aZKgIi3Rin1ZXPLmIkoqWl6JO1WVrZLFO94jt/iw03iH6D6M6DJVpc6bGHZYew/smVkz1n4KnP8p+PibFkvEE7h8KfbIkSNMmTKF9957j9zcXMrKyjAMxymE3Nxcnn/+ea699loyMzObLKxIS/PKz7tcLnVWLy42druNZbs+5EjBPqfxdhFdubD7NKwW3VXiNew2WP1r51KXeC2M+kylTsQFLn83fOWVVygsLGTmzJksXLiwutQBREVF8fHHH1NcXMzrr7/eJEFFWqKc4vIzHwTEhQfRvlVwE6cxh2HY+Tnlcw4d2+U0HhOWyNieN+Fj1R0lXsNeBatugb3v1Ix1vAlGfghWP/NyiXgQl78j/vzzz1x//fWMHj2awsLCWvv79+/PDTfcwNy5cxs1oEhLYBgG6fklVNrsTuNllc5n6/rGRRAZ5HzWIi48mEfH98Zq9b4zdoZhsHrvXPYe3eQ0HhHclgm9bsXPJ6DuF4rnsVfCLzfCwc9rxjrdBkP/C1Yf83KJeBiXi93Ro0fp3r17vcd06dKF7Ozseo8REWeZBaVc8uYitmbknfHY/3fFIC7q3nIWON94cCG7MlY6jYUFRjGxzx0E+HnnGcoWK28bpJ90YqDLb2HIq6DL7CIN4vLfmLCwMHJzc+s95sCBA4SGttxFyEXOxv9Wp7hU6lqa7ek/sSVtidNYkH8YE/vcQbB/uEmppMlEnQcXfA1Wf+h2Pwx5TaVO5Cy4/Ldm8ODBfPLJJxQXF9e5f+3atbz//vsMGTKk0cKJtASHC0pdOs5igV6xEU0bxk2kHFnH2n3fOY35+wYxsfcdhAVGm5RKmly7i+HSjTDo344/8CLSYC5fir333nuZNm0al19+OSNHjsRisfDVV1/x/fffs3nzZnbu3ElgYCD33ntvU+YV8WiGYWAYp445b0cF+xMe6HfKWAAPXtiTeC99QOJk+bZ0tqU4X371tfpzUe/biAyJNSmVNLqqYijPhZAE5/FWvczJI+IlXC52PXr04J133uGpp57iyy8dM3/Pnj27en/Pnj3585//TLdu3Ro/pYiHq7TZufuLVXy8Yf8ZJxZ++pIB3H1+/fezeqvDeSmkVazCoKbtWi0+jOt1MzFhiSYmk0ZVWQjLLoOSQzDhx9rlTkTOWoPmCTjvvPOYM2cO+/fvZ8+ePZSUlBASEkLXrl1JTNQ3XZHT+W7HId5Zk2p2DLd2tPAgS3bMwqDmyWALFkZ3v552EV1NTCaNqiIPll4KOasc20vGOy6/+oaYGkvEW7hc7L7++muGDRtGXFwcHTt2pGPHjrWO+fnnn9m7dy/Tp09vzIwiHm9/bpHLx3aKDmvCJO7pWHEmC7e/Q5W9wml8ZJepdGjdx6RU0ujKc2HpxZC7rmas690qdSKNyOWHJx5//HG2bdtW7zG7d+/mzTffPOdQIi2BxeL8v7AAP+49vzsTurWs+8gKy3JYsP1/VFQ5P0QyJOkyusbqYSyvUZbtODt3cqkb/Ar0eMi8TCJeqN4zdmvWrGHNmjWA46bv77//nuTk5DqPraioYO7cuZSXuzZTvkhL9sCFPfjXlSotJRUFLNj2P0ornCc975cwjt7xF5iUShpd6RFHqcvffnzAAkNnQpc7TY0l4o3qLXaZmZl89dVXHD58GIvFwg8//FDvm1ksFu6+++5GDSgi3qm8soQF2/5HYZnz/JhRPp05L/Eik1JJoys57Ch1BSeWhLPA8Leh061mphLxWvUWu8mTJzN58mRyc3MZOXIk991332nnqbNarbRv357Y2JZ1GUlEGs4w7CzZOYu8kiNO451iBhBc2AWL5jDzDsVpsHgcFO1xbFt8YMT70PEGc3OJeDGXHp6IiopixowZXHLJJXTu3Pm0xxUUFJCWlkZCgh5dF5HT23t0E0cK9jmNtY/swaiu17Bx4yZzQknj2/ncSaXOF87/CBKvMTeTiJdz+eGJGTNm1FvqABYtWsT1119/zqFExHtV2SvZcGCB01ib8I6M6XEjVi327l3Oex7aXQZWP7jgC5U6kWbQoHnsjhw5wuzZs0lPT6eqqsppX3l5OStXrqSysrJRA4qId9l1eCXF5XnV21aLDxd0uxZfH7/Tv0g8k0+Ao9DlboCYkWanEWkRXC52+/bt47rrrqOgoABwPChhnLQWksViwcfHRw9PiMhplVeVsOXQUqex7nHDCQuMMimRNKrSIxDYxnmdV59AlTqRZuRysXvppZcAeOqpp4iPj+fOO+/kz3/+M0lJSaxZs4avv/6av/zlL1xwgaYoEJG6bU370Wm+Oj+fAPonjDMxkTSaY5thyQToNgP6/tnsNCItlsvFbsuWLdx2221MmzaNwkLHnFNdunRhyJAhDB8+nMsuu4zp06fz0ksvMXjw4CYLLNIcsovK+NePO9jXgBUj6rM7q6BR3seTFZfnsePwCqexvu3HEOinVQc8Xs46WDoRKo7B1qfANxR6PmJ2KpEWyeVil52dXb0e7ImpCE6+n65z585cffXVvPTSS8yaNauRY4o0r+tn/cSSPZlmx/AqGw8sxG7U3Jsb7B9Or3bnm5hIGkX2KscyYZXH//Hi1wpiRpmbSaQFc/mp2MjISI4cccw5FRwcjNVqJTPT+Qdfp06d2LFjR+MmFGlmhmGwLPXImQ88B+EB/k36/u7mWHEme7I2OI0NSJyAr0/L+n3wOlk/w5KLakqdfxSMXwKth5mbS6QFc7nY9e/fn/fff59Vq1ZhtVpJSkriww8/pKSkpPqYn376iaCgoCYJKtKc7Cc9GNTYWgX6cd15HZvs/d3R+v0/ADW/p62C2tCl7SDzAsm5O7LUcaau6vjtCgExMH4pRA00N5dIC+fypdg777yT6dOn89ZbbzF8+HCuvvpq/vnPfzJ27Fh69OhBRkYGaWlpTJkypcEhCgoKePnll1m8eDFZWVlEREQwevRoHnzwQWJiYs74+oqKCt544w3mzp1LZmYm0dHRjB49mgceeICoKD1tJ+fuw5sa59KSv48P5yfF0Das5fwDKCMvlUPHnNeYHtTxEqwWzVnnsTIWwPIrwVbm2A5s6zhT16qXublExPVi17dvXz755BNSU1MBuPXWW8nOzuajjz5i9erV+Pr6cumll/J///d/DQpQUlLCTTfdRGpqKjfeeCN9+vRh//79vP3226xatYovvviCyMjI076+qqqKu+66i3Xr1nHzzTfTo0cPduzYwaxZs1i/fj1fffUV/v663CPnZtp5SWZH8EiGYRw/W1ejTXhHEqJ6mpRIzln6d/DTVLBXOLaD2jlKXXh3c3OJCNDACYq7d+9O9+6Ov7wWi4Xf//73PPTQQxw7doyoqCh8fBr+L/BZs2aRnJzMn//8Z264oWb9wJ49ezJjxgxmzpzJY489dtrXf/rpp6xcuZJ///vfXHrppQBceeWVhIeH89VXX7F58+bTrm8rIk1rf/ZWsosOOY0N7nip1oL1VJUFsPLmmlIXnOgodWH1r0okIs3H5XvsTsfX15eYmJizKnUAc+fOJTg4mKuvvtppfMKECcTGxjJ37lyniZBP9eGHH9KzZ8/qUnfCvffey+LFi1XqRExis1ex4cB8p7EO0b1pE97BpERyzvzC4YKvwCcIQpJgwo8qdSJuxqUzdikpKaxbtw5fX1969+5Nr15130dx9OhRnn76aV5++WWXvnhxcTEpKSkMGjSo1uVSi8VC//79mT9/PocOHSIhIaHW648cOUJqaip33nln9Vh5eTl+fn5YrefcWcWD7MkuYOmeI1RW2Rv0uoNpuawudb7/y6DpHpxoSXZnrqGwLKd624KVgR0uMTGRNIq2Y2DM9xDaGUJqf18WEXOdsdg988wzfPjhh05jF198MS+88ILTWbqvv/6af/zjH9VLjrkiLS0NgLi4uDr3x8bGVh9XV7E7cb9fYmIi//vf/5g1axYZGRn4+flx/vnn89hjj5GUpHujvN36tBwufGU+ZVW2s3uDdZqvrrFVVJWxOW2x01i32CG0Cj7zw1DiZqqKwfeUSaTbjjElioicWb3Fbu7cuXzwwQd07dqVK664Aj8/P5YsWcL8+fNJTEzk4YcfJjMzkyeffJKff/6ZkJAQnnjiCZe/eHFxMcBpp0g5MV5UVPfs/3l5eQDVxfP++++nVatWrFq1ig8//JDNmzczZ84c2rZt63Im8Txfbjlw9qXOBf4+OvvbUNvTl1NWWVy97Wv1p3/iBBMTyVlJmQnb/w4TlpmdRERcVG+x++yzz+jQoQNffvll9aXS2267jUceeYQvvviC2NhY/vWvf1FUVMTEiRN54oknaNOmTaOHPN2N1idWvigsLOTbb78lODgYgPHjxxMTE8MLL7zA22+/zeOPP37Gr7Ft27bGC3wa69evb/Kv0RLtO9S0Z9yGx4Xos2uASqOU5LIfncairF3YuXW3y++h32/zxRz7hMSs5wEo//58/BL/q8/FDekzcU9mfi71Frvk5GR+/etf17r/7dZbb+W7777jr3/9K3FxcTz//POMGTOmwV88LCwMqDlzd6oT46GhoXXuP1HkxowZU/3fJ1x11VW88MILrF271qUsffr0ISAgwKVjz8b69esZNEgTsjaFNgfXwO7c6u1RSW3oGxfh0muPHj1a71yJnVuH8ethXQkL9DvXmC3GL3tmY2TWnEEN9AvlokHX4efr2t8v/V1xAztfgOTnqzcDwuOxW4P0ubgZ/V1xT039uZSXl9d7MqreYldYWEj79u1rjZ8Yu+aaa3j88cfPerWJhIQELBYLGRkZde5PT08HoEOHup+iO5GjrgcloqOjsVgspy2N4r2u7p/IfRe4Nk+avjE2rvySo6RkOv9jqn/CeJdLnbiB7X+HzX+s2W49Asb8gG3rHvMyiYjL6r15yDAMfH1rd78TY1dcccU5LSEWFBREz5492blzJ2VlZU77bDYbmzZtIj4+nnbt2tX5+i5duhAWFkZycnKtfYcPH8YwjCa5NCwidVt/YB4GNU8mhwVG0z12qImJxGWGAVueci51bS6EsfPBv5VZqUSkgUy/K/yqq66irKyMTz75xGl8zpw55ObmMnXq1Oqx1NTU6idpAfz8/Jg8eTJr1qxh3bp1Tq//4IMPABg9enQTpheRE7IKDnAwZ7vT2KCOF2O1aukwt2cYsPn/YNtfasbajnNMa+IXZl4uEWmwBq080RSmTZvGt99+y3PPPUd6ejp9+/YlJSWFd955hx49enD77bdXHztp0iSSkpKYN29e9diMGTNYvnw5v/3tb7n99tuJjY3ll19+Ye7cuXTv3p0bb7zRjF+WSItiGAbr9n/vNNY6NIEO0X1NSiQuMwzY8Agkv1gzFneJYyJi35azprGItzhjsatv6Z/GWBbI39+ft99+m1dffZV58+bx8ccfEx0dzbRp07j//vtrPRRxqqioKD799FNeeuklPvroI/Ly8oiJiWH69Oncd99953SpWERck5a7g6yCA05jg5O0dJhH2Pe+c6mLvwJGfQ4+ui9SxBOdsdg9/PDDdT6cYLFYuO2222rts1gsbNq0qUEhQkNDefTRR3n00UfrPa6ue+nA8aDE008/zdNPP92grysi585u2Fi/f57TWPvIHsS26mRSImmQjjdA+jeQ9hUk/ApGfgQ+/md+nYi4pXqL3ekeWhAROWHPkfXklx6t3rZgYVBHLR3mMax+MPJjSHkVut0HVtPv0BGRc1Dv3+AlS5Y0Vw4Rl5RWVlF+ynqwp25L86m0VbDx4EKnsc5tBhIZEmtSIjkjuw0sVjj5MrmPP/R4yLxMItJo9E8z8QgVVTamf7SCL7ccxG4YZseR43Yc/pnSisLqbR+rL+d1mGhiIqmXvRJ+uRFCO0H/fziXOxHxCip24hF+2HWYzzcfOPOB0mzKKovYdsh56bCe7c4nJEBznrklWzmsuA4OzXFsWwOg31/qf42IeBwVO/EIacdcX0GkW4yKRXPYfHAJlbby6u0A32D6th9jXiA5PVsZ/PQrOHzSlDSV+Y6pTnTWTsSrqNiJR/LzsRLs5zzxbYi/LzcN6sRF3eJMStVyFJblkJy52mmsX8JYAjTvmfupKoHlUyDzpHshe/4BBjyrUifihVTsxCPdNbwr/5mqparMsmH/AuyGrXo7JCCCHnEjTEwkdaosgh+vgKxlNWO9n4B+T6vUiXgpFTsRaZDswkPsy97sNDaww8X4aJoM91JZAMsmwdEVNWN9n4a+T5qXSUSa3FmtFWsYBrm5uVRWVjZ2HhFxY4ZhsH7/D05jUSFxdIrpb1IiqVNFHiyZ6FzqBvxTpU6kBWhQsUtNTWXGjBkMHDiQUaNGOa0w8cwzz7B79+7GzicibiQ9bzcZ+alOY4M6XorFclb/RpSmYBiOByVyTroHcuCL0OsP5mUSkWbj8nfj1NRUrr32WpYuXUqHDh0wTppLLDc3ly+//JKbbrqJ1NTUet5FRDyV3bCzfp/z2bq4iC7ER3YzKZHUyWKBfn8F31DH9uBXoceDpkYSkebjcrF77bXX8PPz46uvvuL99993KnZRUVF88803+Pv78/rrrzdJUBEx196sjRwryXQaG9zxUpPSSL1iRsKY72DYW9DtHrPTiEgzcrnYrV69mptuuonu3btjqeNpqoSEBG688UbWrl3bqAFFxHxV9ko2HlzgNNYpZgDRofEmJRInda3G0uZC6HxH82cREVO5XOzy8vJITEys95j27duTm5t7zqFExL3sOvwLxeX51dtWi4+WDnMXxQdh0YVQkGx2EhFxAy4Xu8jISDIyMuo9ZufOnURGRp5zKBFxH+WVJWxJW+o01iNuOGGBUSYlkmpF+2DRaDj6MyweB4W6x1mkpXO52I0cOZJZs2aRmZlZa5/NZuPrr7/mww8/5Pzzz2/UgCJiHpu9ig0H5lNhK6se8/MJoF/COBNTCQAFKY4zdcX7Hdvl2VComQlEWjqXZxS97777+PHHH7nsssvo378/FouFmTNn8t///pcdO3aQk5NDREQEM2bMaMq8ItIMCsty2Z25hpQj6yirLHLa17f9GAL9QkxKJgDk74Ql46H0+FUUawBc+DW0u8TUWCJiPpeLXfv27fnss8949tlnWb58OYZh8PPPPzvexNeXCRMm8Ic//IH4eN1MLeKJ7Iad9GPJJGes4tCx3UDtG/KD/cPp1U5n5U2Vt81R6sqyHNs+QTB6LsSONzeXiLiFBq0BlJiYyGuvvUZZWRn79u2jpKSEkJAQkpKSCAgIaKqMItKESioKSTmylt2Zayguzzvtcb5WP87vejW+Pv7NF06c5W6EpRdBeY5j2zcERn8HbUebm0tE3IbLxe7xxx9n8uTJDB8+nMDAQHr27NmUuUSkCRmGQWb+XpIzV3MgZxuGYT/tsX4+AXRpM4ge7UbQKiimGVOKk5y1jmXCKvMc237hMOYHx5x1IiLHuVzsZs+ezddff01MTAyXX345kydPpkePHk2ZTUQaWXlVKalH1pOcuZr80qP1HhsdEk/3uGEkxQzAT2fpzFWQDEsmQGWBY9svAsYtgOghpsYSEffjcrH76KOP+OGHH1i4cCFvv/0277zzDl26dGHy5MlcfvnlxMXFNWVOETkH2YWHSM5cxd6jm7HZK097nI/Vl6TW/ekRN5zWYQnNmFDqFdoF2l0OBz4C/ygYtwiizjM7lYi4IZeL3cCBAxk4cCB//OMf2bBhQ3XJe+GFF3jxxRcZNGgQV155JZdccgmhoaFNmVlEXFBlq2Dv0c0kZ64ipyi93mPDg2LoETuMzm0HEuAb3EwJxWVWHxjxnuPya9e7IbKf2YlExE016OGJE04ueevXr2f+/PksWrSIJ598kmeeeYZNmzY1ckwRcVVeSRbJGavYk7WBypPmnzuVxWKlQ3RvuscOJ7ZVpzqXChQ3YvWFoVqLW0Tqd1bF7mQDBw7EYrEQHBzMF198QU5OTmPkEpEGsNmrOJiznV0ZqzhSsK/eY0MCIugWO5SubQcT7B/eTAmlQQ7Nhawf4bz/ByrcItIAZ1XsbDYbq1evZv78+SxevJicnBwsFgtDhgzhgQceaOyMInIahWW5pGSuZfeRtbUmEnZmIT6yGz1ihxEf1QOrxeVFZ6S5pX0FP18HRpWj1A14TuVORFzmcrGrrKzkl19+qS5zBQUFGIZBnz59+PWvf82kSZNo06ZNU2YVEU5MJLz7+ETCydQ1kfAJgX4hdG07hG6xQ7W2qyc48Cn8ciMYNsd22lfQ+//AX2twi4hrXC52I0eOpKioCMMw6NSpE9OnT+eKK64gMTGxKfOJyHGlFYWkHFlHcubqeicSBmgb3pHuccPpEN0HH+s533EhzWHfLFh1K5yYUzCsG4xfrFInIg3i8nf8kJAQrr32Wi6//HJNTizSTAzD4EjBPnZlrOJgznbsJ87k1MHPJ4DObQbSPXY4kSFtmzGlnLPUt2H1r6k++xre01HqgjSNlIg0jMvFbtmyZU0YQ0ROVl5VSmrWBpIzVpNfmlXvsVEh7egRN5ykmP74+WhpP4+T8gasvbtmO6KvY566QN3aIiINd9pi98orr3DppZfSuXPn6m1XWCwW7r333sZJJ9LCnJhIeN/RzVS5MJFw97jhtA5tr6lKPFXyf2D9SQ+cRZ4H4xZCQLR5mUTEo9Vb7Lp166ZiJ9LEqmwV7Du6mV2Zq8kpOlTvseFBrekeO4wubQYR4KeJhD1a8svOpS56KIydp3vqROScnLbYvf/++3Tp0sVpW0QaT15JFsmZq0k9sp6KM0wknBjVmx5xw4ht1Vln57xFzEjwawWV+dB6JIz9wbGyhIjIOThtsRs6dGi923XJzc2lpKTk3FOJeCnHRMI7SM5cRWb+3nqPDfZvRffYoXSNHaKJhL1R1CAYOx+2/w1GfgR+WopRRM6dyw9P9OzZk5deeomJEyee9pjZs2fz2WefMX/+/EYJJ+ItisqOsTtzjYsTCXele+xw2kd1x2rxabaMYoLWw2D0N2anEBEvUm+xKyoqoqCgAHBMu3Ds2DEOHz5c57Hl5eWsW7eOrKz6n+ATaSnshp3Dx3azK2MV6ceSMeqZSDjAN4SusYPpHjuUsEDdOO91DAO2PgWJ10BEH7PTiIgXq7fYvfvuu7zyyitYLBYsFgtPPfVUvW9mGAaDBw9uzHwiHqe0ooiUI2vZnbmGovJj9R7bJrwjPWKH06G1JhL2WoYd1t0PKa/Cnjdg/I/QqofZqUTES9X7k+T2229nyJAhbNmyhRdeeIGhQ4cSHx9f57FWq5X27dtz3XXXNUlQEXd2YiLh5IzVHMjZ5uJEwsOIDIltxpTS7Aw7rPkNpL7l2C7Lgt3/gSGvmZtLRLxWvcUuODiYYcOGMWzYMD7++GPuvPNOLrjggubKJuL2KqrKSM1aT3LmavJKzjSRcBzd44bTKWaAJhJuCew2WH0H7HuvZqzD9TDoP+ZlEhGv5/K1nyVLljRlDhGPkl10iOSM1ew7uumMEwl3bN2PHnHDaR2aoKlKWgp7FaycDgc+rhlLmg7D3garHogRkaajlSdEXFRlq2Bf9haSM1aRfaaJhANb0z1OEwm3SPZKWHEDpH1RM9b51zB0Jlis5uUSkRZBK0+InIHLEwljJTG6F93jhhPXqhMW/RBveWzl8PO1kH7SFCZd74HBL6vUiUiz0MoTIqdxKDeZbek/ujCRcDjdYofSre1QggM0kXCLZSuH5VdBxg81Y90fgoEvgC7Bi0gzadSVJ0S8QVFZHmv2fsPB3B31Htcuohs94obRPqqHJhIWsPpBUFzNdq9Hof8/VOpEpFk1aOKs0tJS9u3bR69evarHNmzYwMKFC/Hz82PKlCl06tSp0UOKNAe73caOwz+z6eCi0z4QEeAbTNe2g+kWO4zwIE0kLCexWGHom4577EKToO9TKnUi0uxcLnYZGRnceOONdOnShTfffBOA7777jt///vfY7XYAPvjgA7744guVO/E4R/L3szJ1NnklR+rc3ya8A92PTyTsa/Vr5nTiMaw+MOJd3U8nIqZx+bvPa6+9RnFxMbfddlv12AsvvEBoaChvvfUW7733HuHh4bz11ltNElSkKVQZ5axI+YIftr5RZ6lrHZbA5f1nMKnf3XRuc55KndSoOAY7/umYhPhkKnUiYiKXz9itWLGCm266iREjRgCwdetWDh8+zIwZMxg1ahQAN9xwA59//nnTJBVpRIZhJ+XIenaXzcNWVlFrv79PIIM6Xkq32CF6ulVqK8+BJRfBsY1QfBAGv6LLriLiFlwudtnZ2SQlJVVvr1ixAovFwtixY6vH2rVrR1ZW/bPvi5jtWHEmK/fMJqvwQJ37O7cZyOCOkwjyD23mZOIRyrJgyQTI2+rYTnkNOt4AMeebm0tEhAYUu7CwMIqKiqq3V6xYQWRkJL17964eKykpITAwsHETijSSSls5mw4uYkf6Cgzstfa3CmrD8M5XEhfR2YR04hFKM2DxeCjYeXzAAsPeUqkTEbfhcrHr3LkzX3/9NZdccglr1qxh3bp1XHPNNU7HLFu2jISEhEYPKXIuDMPgYM52Vu+dS0lFfq39PlY/+ieMo3f8BfhYG/SguLQkJYdg8TgoTHFsW6ww/D1IusncXCIiJ3H5p9j06dOZMWNG9T12AQEBTg9SPPbYYyxdupTHHnus8VOKnKXCslxWp37DoWO76twfZo1j4sCbCQuMauZk4lGKDzhKXdHxyaotPjDyQ+hwnbm5RERO4XKxmzBhAv/617+YO3cufn5+3HLLLU733O3du5drr72WW265pUmCijSEzV7F9vSf2Jy2BFsdc9KFBLRiaKfJHN1XqlIn9SvaC4vGQslBx7bVD87/FBKuMjeXiEgdGnTdadKkSUyaNKnOfbNmzSIgIKBRQomci4y8VFalfk1+6dFa+ywWK73bjaJ/4nj8fALI3r/ehITiMQpSYPFYKE13bFv94YIvIf5yc3OJiJxGg28oqqioYP369Rw8eJCSkhJCQkLo1KkT5513XlPkE3FZaUUR6/Z9R+rRjXXubxPekRGdpxAZEtvMycRj2csc/wPwCYQL50DcRHMziYjUo0HF7vPPP+f555+noKAAcNyUbjk+d1ObNm3405/+xPjx4xs/pUg9DMNOcuYaNuyfR4WtrNb+AN9gBne8lC5tB2lOOmmYiL4wdiH8eAWMeB9ix5mdSESkXi4Xu0WLFvHkk0/SunVrrr/+epKSkggKCqK0tJSUlBQWLFjAAw88wHvvvcegQYOaMrNItZyidFbu+ZrsorQ693dtO5hBHS8l0C+kmZOJ14g6DybvcZyxExFxcy4Xu/fee4+OHTvy6aef0qpVq1r7H3roIa6//nrefPNNZs6c2aghRU5ltVSyOvUbdmWsxMCotT8yOJbhXabQNrxj84cTz5WzFiy+jjJ3MpU6EfEQLl+X2rlzJ1dffXWdpQ4gMjKS66+/nk2bNjVWNpE6GAyOz6dX1EJ2ZvxSq9T5Wv0Z3HESVwy4T6VOGuboL47Jh5deVLOqhIiIh3H5jF15eTlt27at95jo6GhKSkrOOZS0XHmlFczblc6xUuf1W3/al0WbkHJu7J9Bn7bFdb42Mbo3wzpdQUhARDMkFa+StRyWTYKqYqgCVkyDS7eA1cfsZCIiDeJysYuNjWX9+vVcccUVpz1m48aNxMbqiUM5O2WVNka9PI+dR5xXh/C12pnULZunx2fj51P7smtoQCTDOl9JQlSP5ooq3iRzsePhCFupYzuwjWOeOpU6EfFALhe78ePH88EHHxAfH8+0adMICwur3pefn8+nn37Kp59+qgmK5aytTcuuVep6xRRx44AMYkMrah1vtfjQJ/5C+iWMxdfHv7liijc5PA9+ugpOPE0dFAfjFkOrnubmEhE5Sy4Xu3vuuYcVK1bwwgsv8OKLLxITE0NQUBAlJSVkZ2djt9vp3bs399xzT1PmFS9WXFFV/d+tAiq5rm8mwxIK6jzW3y+eSX2vIyK4TXPFE29zaC78fDXYj/+jIbg9jFsC4V3NzSUicg5cLnbh4eF8/vnnvP/++yxevJi9e/eSnZ1NcHAw/fv35+KLL+aGG27A319nTuTchAdU8scx+4gOrr0UWKU9gOjwC7mq/7jqORRFGuzgl4776Izj/5gI6QDjl0JoUv2vExFxcw2aoDgwMJC77rqLu+66q6nyiHB59+w6Sp2F7rHDGNhxIgG+wabkEi+x/2NYeTMYNsd2aCdHqQtJNDeXiEgjcLnYFRYWUllZSVSUFkyXpuPvY2dEYp7TWFRIO0Z0uYqYsARzQol3yV5VU+rCusH4JRAcb24mEZFGcsZit3r1ap555hn27NkDQFxcHA8++CCTJ09u8nDS8gxrn0+wn716O9AvlMv634OPtcHLGovUbdCLYCuB7F8cD0oE6Ul+EfEe9f60TE5O5te//jV2u51u3boREBBASkoKjz76KFarlcsvv7y5coqXOVpUxpHCUqex/bnFjEnKdRrr2nawSp00LosVhs6EynzwjzQ7jYhIo6r3J+abb76Jn58f77//Pn369AHg6NGj3HXXXfzrX/9SsZOz8n/fbeCfS7bXGu8QUcqfxpZVbxsGdIsd2pzRxBsdngexFznPS2exqtSJiFeqd0mxdevWce2111aXOoCYmBgefPBBMjIyOHDgQJMHFO+SX1rB/1u6o859p56tyyhuTVig7umUc7DjOVh2Kay5Cwz7mY8XEfFw9Ra7nJwcunXrVmu8R48eGIZBTk5OkwUT75RdXI7dqL16RJCfjWHtnScnNixaSULOwda/wqZHHf+9923Y9W9T44iINId6L8VWVVURGhpaazw42DHdhM1ma5pU0mL4+VjpFhPGgNgMAnxrCl+5LYgHxkw0MZl4LMOALX+C7c/UjLUZA100TZOIeL96z9iJNLWEiGA2/+4KpvYudhofnjSK0ABNdi0NZBiOs3Qnl7rYi2DMd+BX+x+pIiLeRo8biumyCvaTV5JVvW3BStfYISYmEo9kGLDhIUh+qWas3SS44EvwCTQvl4hIMzpjsdu4cSNVVVVOY2VljicXV69ezdGjR2u9ZtKkSY0UT1qCXZmrnLYTo3sR7B9uUhrxSIYd1s2AlNdrxtpfCed/Cj4B5uUSEWlmZyx277777mn3vfrqq07bhmFgsVhU7MRlwX6VHMje5jTWPW6YSWnEI9ltsPY3kPq/mrHEa2Dkh2D1My+XiIgJ6i12M2bMaK4c0kL1a3sUu1HzEE5YYDRxrTqbmEg8TulhSJ9bs93hBhjxHmhiaxFpgVTsxDQWDAbGZTmNdY8dhsWiZ3qkAUISHEuDLR4D8VfA0LecJyMWEWlB9E9aMU2vNsVEBpVXb1stvnRpO8jEROKxIvrAxesgJNGxqoSISAul74BimlNXmujYui+BfiEmpRGPYSuHvG21x0M7qtSJSIunM3bSJHYfLeCxbzewL6fIaby8ynE/XWRgJf1jC5326aEJOaOqUvjpKsheBeMXQ5TO8IqInEzFTprEjR/8xIZDuafdf0HHY/icdHIlMjiWNmEdmiGZeKyqYvhxMhxZ4thechFcth2C4szNJSLiRlTspElsSj922n1Wi8EFHZ33d48bhsViaepY4qkqC2HZZXD0p5qx7g9AYKx5mURE3JCKnTQJA+O0+/rHFhIVVDPpta/Vn04x5zVHLPFEFfmw7FLIXlkz1v/v0Ptx8zKJiLipBhe73NxcVq1axeHDh7nsssuIi3NcBikpKSE4OPisQhQUFPDyyy+zePFisrKyiIiIYPTo0Tz44IPExMQ06L3Ky8uZPHky+/fv5/3332fYMN235Q7WPXQZ1uOXXvdkfE5hac2+TjED8PfVkk9Sh4pjsORiyF1bM3beC9DzYfMyiYi4sQYVu9dee4033niDyspKAPr3709cXBw2m41LLrmE6667jnvvvbdBAUpKSrjppptITU3lxhtvpE+fPuzfv5+3336bVatW8cUXXxAZGdmgjPv3729QBml6/dtFYrVaKCzLYePefU779NCE1KksG5ZeBMc21YwNehm6a35NEZHTcXlugLlz5/Kf//yHnj178sgjj2AYNZfaSkpK6NmzJ6+88gpz586t511qmzVrFsnJyfzxj3/k//7v/5g8eTL3338//+///T8OHTrEzJkzXX6v5ORk/ve//9GzZ88GZZDmk5yxxmm7dWgC0aHxJqURt1WWBYvHOpe6oTNV6kREzsDlYvfhhx9y3nnn8cknn3Ddddc57QsLC2PmzJkMGTKEDz/8sEEB5s6dS3BwMFdffbXT+IQJE4iNjWXu3LlOJfJ07HY7Tz75JPHx8UybNq1BGaR52OxVpBxZ5zSms3VSp32zIP/EXHUWGPY2dLnL1EgiIp7A5UuxKSkpPPzww/U+uXj55Zfz/PPPu/zFi4uLSUlJYdCgQfj7+zvts1gs9O/fn/nz53Po0CESEhLqfa8PPviALVu28P7773Po0CGXM8jZKy6v5IMN+9h6uPYTsHV18QM52yivKq7e9vcJJKl1v6aMKJ6qx8NQfBBSXoHh70PSjWYnEhHxCC4Xu6qqqjM+HOHr60tFRYXLXzwtLQ2g+gGMU8XGxlYfV1+xy8jI4MUXX+Saa65h6NChKnZNzDAMvtp6kEfmrCMtr8Tl1yVnrHba7tx2EL4+/qc5Wlo0iwUG/RuSbobowWanERHxGC4Xu8TERNatW8dVV1112mO+/fZbEhMTXf7ixcWOszdBQUF17j8xXlRUVOf+E5566ilCQkL4/e9/7/LXPtW2bXUsUdTI1q9f3+Rfo6kdKCjn+XWZrM4sPvPBxwX4WPhl3VKOVDg/NFGVE8r6Y+b/nnjD5+Lp/CqzqPSNAovjW1LNZ2KB/fp83IX+rrgffSbuyczPxeViN3nyZP7973/TtWtXxo4dCzjO3OTm5rJ582bee+89Vq9efU7l6nTqu/z73XffsWzZMl566SXCw8PP+mv06dOHgICAs379maxfv55Bgzx3+aPi8kr+vngbLyzbR6XN3qDXTh/ShVbtymF/zVjb8CTO7ze2cUOeBU//XLxCQTIsvhLajIYR77N+4yZ9Jm5If1fcjz4T99TUn0t5eXm9J6NcLna33347mzdv5tlnn+Wf//wnALfeemv1gw2GYXDRRRdx6623uhwuLCwMqDlzd6oT46GhoXXuz8/P5+9//ztjx47lkksucfnrSsPM2ZbGA7PX1HnZ1ddq4bcju9E9plWdr02KDuWibnFsSVvoNN4uokuTZBUPk78DFo+DsiNw4CPwCwOfO81OJSLisVwudj4+PrzyyissWrSIH374gT179lBaWkpISAhdunTh0ksvZdy4cQ364gkJCVgsFjIyMurcn56eDkCHDnWvIfrPf/6TkpIS7rnnHjIzM6vHCwoKAMdkypmZmURFRdV6OENc88XmA1z3/vI6943t0pb/XDWUXrERDX9jLR8mx7bAkglQftSx7RMMiddCurmxREQ8WYNXnpgwYQITJkxolC8eFBREz5492blzJ2VlZQQG1qw+YLPZ2LRpE/Hx8bRr167O169atYqSkhKuueaaOvc/+OCDAFqB4hx8vvlArbG48CCenzyI6wZ01PqucnZyN8CSi6Ai17HtGwpjvoc2F0C67hkSETlbpq8Ve9VVV/G3v/2NTz75xOky7pw5c8jNzeW+++6rHktNTcXf37/6Cdm//e1vlJWV1XrPlStX8t577/Hwww/TrVs3unXr1uS/Dm9VWlnltH3LkM68NGUIYYF+JiUSj5e9GpZeDJX5jm2/cBg7H1oPNzeXiIgXcLnYjR8//ozHGIZBVVUVy5fXfemuLtOmTePbb7/lueeeIz09nb59+5KSksI777xDjx49uP3226uPnTRpEklJScybNw+AESNG1Pmex4455lUbMGCAztQ1sqv6JqjUydnL+hmWTYKqQse2fySMXaApTUREGonLxe50qz9UVlaSnZ2NYRh07969wU+m+vv78/bbb/Pqq68yb948Pv74Y6Kjo5k2bRr333//GefOExEPcWQZ/Hg5nJikOqA1jFsIkQPMTCUi4lVcLnZLliw57b6Kigq+/PJLZs2axXPPPdfgEKGhoTz66KM8+uij9R6XnJzs0vtNnTqVqVOnNjiHiDQRexWsuaum1AW2hXGLIKKPublERLyMy2vF1sff35/rr7+eKVOm8I9//KMx3lJEvInVF0bPdRS6oDgYv0ylTkSkCTRKsTthyJAhrF69+swHikjLE94dxi+BCcuhVQ+z04iIeKVGfSr26NGj2Gy2xnxLEfFUFXngH+E81qqXGUlERFoMl4vd2rVrT7uvsrKSffv28eabb9Kli1YUEGnx9n8Ea+91zE0XU/fT6yIi0vhcLnY333xzvZPRGoZBQEAAL7zwQqMEExEPtfc9WHUbYMCySxyXX6O0nqWISHNwudjde++9py12vr6+xMTEcMEFF9CmTZtGCyciHmbPf2HNb4Dj0yMFJ0BQe1MjiYi0JC4Xu5NXgBARqWX3q7BuRs12RH/HPHWBMeZlEhFpYVx6KraiooIJEyZUr/ggIuJk14vOpS5qkOMSrEqdiEizcqnY+fv7U15eTmZmZlPnERFPs+OfsOHhmu3o4Y7JhwOizMskItJCuTyP3cMPP8x7773HunXrmjKPiHiSrX+FTY/VbMeMgnELak9zIiIizcLle+w2bNhAly5dmD59OjExMbRr147Q0NBax1ksFt58881GDSkibihzMWz9U81227GO1SV8Q8zLJCLSwrlc7D7//PPq/z5y5AhHjhyp87j6pkQRES/Sdhz0etRxKTb2Irjwa/ANNjuViEiL5nKxW7x4cVPmEBFPY7FA/39AWFfoeCP4BJqdSESkxau32E2fPp3777+fwYMHEx8f31yZRMQdGXYwbGD1qxmzWKDzHeZlEhERJ/U+PLFmzRpyc3ObK4uIuCu7DVbfASumgb3S7DQiInIaLj8VKyItlL0KVt0Ce9+FtK9g5XTHmIiIuB2X77ETkRbIXgm/3AgHax6ewicY0ENSIiLu6IzFrrCwkJycnAa9aXR09FkHEhE3YauAFdfBoa9rxrr8Foa8Chad7BcRcUdnLHZPPPFEg97QYrGwY8eOsw4kzc8wDF7+aRff7jhEpc3utG9rRp45ocRctjL46Wo4/F3NWLf7YdC/HQ9MiIiIWzpjsevcuTORkZHNkUVM8vnmAzw0RyuKyHFVJbD8KshcUDPW83cw4DmVOhERN3fGYnf//fczceLE5sgiJllzMNvlY6ODA5owiZiuqhh+vAKOLK0Z6/1H6PdXlToREQ+ghycEw3DtuHFdYhmS2Lppw4i51vzGudT1fRr6PmleHhERaRAVO6nl7pHduGZAR6exyCB/+sRGYLXqrI1X6/dXyPoRSg7BgGcdS4aJiIjHULGTWjq3DmN057ZmxxAzhCbBuCVwZDF0/a3ZaUREpIHqLXYzZsygc+fOzZVFRJqbYdS+dy68q+N/IiLiceqdjErFTsSLlR6BBcPhyDKzk4iISCPRLKPS5Ox2G2nHdjmNWS0+JqURAEoOw+IxkLMGfrwcjq4wO5GIiDQCFTtpclsPLeNYcYbTWFyrTialEYrTYNFoKDhetm1ljjEREfF4enhCmlReyRE2py1xGusUM4DWYQkmJWrhivbB4nFQvN+xbfGF8z+GxKtNjSUiIo1DxU6ajN2wsyLlS+yGrXos0C+EoZ2uMDFVC1a4x1HqSo6fnbP6wajPof2V5uYSEZFGo2InTWbn4V84WnjQaWxYp8kE+oWYlKgFy98FS8ZB6fFL4tYAuOAriJ9kbi4REWlUKnbSJArLcth4YL7TWEJULzq27mdSohYsbxssGQ9lWY5tnyC4cA7EXWRuLhERaXQqdtLoDMPgl5SvqLJXVo/5+QQyovMULFpvtHmVZTmefi3PcWz7hsDob6HtGDNTiYhIE9FTsdLoUo6sJSM/1WlsaNJlBAeEm5SoBQtsA11nOP7bNwzGzlepExHxYjpjJ42quDyftfu+cxqLi+hCl7aDTUok9P0zWHwgbiK0HmZ2GhERaUIqdtJoDMNgVerXVNrKq8d8rX6M7DJVl2DNZLFA3yfNTiEiIs1Al2Kl0ezL3kxa7k6nsYEdLyEsMMqkRC3QkaXw83VgqzA7iYiImEBn7KRRlFUWsTp1rtNYm7AO9IwbYVKiFihjASy/0rGShL0CRn3mmKtORERaDJ2xk0axOnUu5VXF1dtWiy8ju/4Ki0V/xJpF+nfw4xWOUgeONWBLM+p/jYiIeB391JVzdjBnB/uyNzuNDUgcT0RwG5MStTBpX8NPVznO0gEEJ8KE5RCSaGosERFpfip2ck7Kq0pZlfq101hUSBx94i80J1BLc/Bz+PkaODFnYEgSTPgRwjqbm0tEREyhe+zknKzb9z0lFQXV2xasnN/1aqxWHxNTtRD7PoRV08GwO7bDusK4xRCSYG4uERExjYqdnLXDeXtIObLWaaxv+9FEh8ablKgF2fsurLodMBzb4T1g/BIIijMzlYiImEyXYuWsVNoq+CXlK6exVkEx9EscZ1KiFmT/R7DqNqpLXas+MH6ZSp2IiKjYydnZeGA+ReW5J41YOL/r1fhqeo2mF3OB4146gMgBMH4pBLU1NZKIiLgHFTtpsKyCA+w4/IvTWM+4EbQJ72BSohYmJMFx2TVhquOeusDWZicSERE3oXvspEFs9ipWpHxJ9WVAIDQgkoEdLzYvVEsU2hEu+NLsFCIi4mZ0xk4aZHPaYvJLs5zGRnadip9PgEmJvJxhwLZn4PA8s5OIiIgHULETl+UUHWZr2o9OY13bDqZdRFeTEnk5w4DNf4QtTzomIM5cbHYiERFxcyp24hK73caKlC8wsFePBfmHMTjpMhNTeTHDgI2/gx3/cGzbyiDlNXMziYiI29M9duKSbek/kVt82GlsROcpBPgGmZTIixl2WP8A7H6lZqzd5TDyQ/MyiYiIR1CxkzPKLznKpoOLnMY6tu5HYnRvkxJ5McMOa34Lqf+tGUuYCiM/Bh9/83KJiIhHULGTehmGnRV7vsBuVFWPBfgGM7zzZBNTeSm7DVbfAfveqxnrMA1GvA+aH1BERFygYif12pWxkqyCA05jwzpNJtAv1KREXspeBStvgQMf1Yx1vBmGvwNad1dERFykYienVViWy/r9853G2kf2ICmmv0mJvJRhh19ugIOf14x1vgOGzFSpExGRBtFTsVInwzBYuWc2VfaK6jE/nwBGdLkKi8ViYjIvZLFC1OCa7a53w9A3VepERKTBdMZO6rQnaz2H81KcxgYnTSIkoJVJibxcrz+AvQLKs2Hgi6DyLCIiZ0HFTmqxUMLavd86jcW26kS3tkNNStRC9HnCMX+dSp2IiJwlXYqVUxgEW1dTYSurHvGx+jGyy690CbaxVBbBpv9zTDp8Kv0ei4jIOdAZO3HSM6YYf0ua09jADhMJD4o2KZGXqSyAZZPg6Ao4tgkunA1aZ1dERBqJztiJkz5ti5y2W4cl0LPd+Sal8TIVebBkoqPUAWT8AOnf1vsSERGRhtAZO3HiZzWctjvFDMBqUf8/Z+W5sHQi5K6vGRv4IiT+yrxMIiLidVTspF6646sRlB2FJRMgb0vN2OBXods95mUSERGvpGIn0pRKM2HJeMjfcXzA4pijrsuvTY0lIiLeScVOpKmUpMPicVC427FtscKwd6DTdHNziYiI11KxE2kKxQcdpa4o1bFt8YERs6Dj9ebmEhERr6ZiJ8cZnBdXyKB2BWYH8Q5Wf0eZA7D4wvmf6EEJERFpcip2gp+1mPtHHKR/bFGtfQG+ISYk8gJBsTB+CSy5CAY8C+0nm51IRERaABW7Fsxmr2J7+k90i1iE1WKrtT8yOJb2UT1MSOYlguNh0haw6q+ZiIg0D/3EaaEy8lJZlfo1+aVHsZ4yp4lhWOjb/kL6J47Hz8ffnICeJm8bFB+A+Mucx1XqRESkGemnTgtTWlHEun3fkXp0Y537d2cH0zZyPIOTtNqEy45tcsxTV1kAF86BdpeanUhERFooFbsWwjDsJGeuYcP+eVTUsfh8YbkPn29ry4qDETw/OdKEhB4qZ51jRYmKY47tVbfC5L2gexNFRMQEKnYtQJWtkkU73iEzf2+d+3PLknhqsT/FFfrj0CBHV8KySxxn6gD8ImD0typ1IiJiGi0C2gJsPLigzlIXGRzLpH53k148SKWuobKWO87UnSh1/lGOp2Cjh5ibS0REWjT9NPdyVbYKUjLXOo35Wv05r8NF9Gw3EqvFBzhqTjhPlbkEfrwCbCWO7YAYGL8YIvqam0tERFo8FTsvty97i9M9dQG+wUw+735CAiLMC+XJDs+Hn6bAid/TwFhHqWvVy9RYIiIioGLn9ZIzVzttd207WKXubKV/Cz/9CuwVju2geMfl1/Bu5uYSERE5TsXOi+UUpZNdmOY01i12mElpvEDR/ppSF5wIE5ZCaCdTI4mIiJxMxc6LnXq2rl1EV8KDok1K4wW6zwB7OaS85jhTF9LB7EQiIiJOVOy8VEVVGXuzNjmNddfZunPX8xHo8hvwCzU7iYiISC2a7sRL7T26iaoTlw2BYP9wEqJ7mpjIAx36BiqLao+r1ImIiJtSsfNChmGQnLHKaaxr2yHHpzYRl6S8AcuvhOWToarE7DQiIiIuUbHzQkcLD3KsJLN624KFbrFDTUzkYZL/A2vvdvz3kaWw+f/MzSMiIuIiFTsvdOrZuvZRPQkJaGVSGg+z4//B+gdqtqOHQt8/m5dHRESkAdzi4YmCggJefvllFi9eTFZWFhEREYwePZoHH3yQmJiYM75+3bp1zJw5k507d1JcXExCQgKXXHIJt99+O4GBgc3wK3AfZZXF7Mve6jTWI264SWk8zLa/wZYnarZbj4SxP4BfuHmZREREGsD0YldSUsJNN91EamoqN954I3369GH//v28/fbbrFq1ii+++ILIyMjTvv7777/n4YcfpmPHjvz6178mNDSU5cuX89JLL7F8+XI++ugjrNaWc2IyNWsDdqOqejs0IIp2EV1MTOQBDAO2PgXbnq4ZazMaRn+rByVERMSjmF7sZs2aRXJyMn/+85+54YYbqsd79uzJjBkzmDlzJo899lidr62oqODJJ58kLi6Ozz//nLCwMACuvvpq7rvvPhYsWMDy5csZM2ZMc/xSTGcYdpIznOeu6x43FIul5RTbBjMM2Pw47PhnzVjsBLhwDvgGm5dLRETkLJj+E3/u3LkEBwdz9dVXO41PmDCB2NhY5s6di2EYdb42Ozubiy66iLvuuqu61J1wwQUXALB79+6mCe6GMvL3UlCWXb1ttfjQpc1gExO5OcOg/dEXnUtd3KUweq5KnYiIeCRTz9gVFxeTkpLCoEGD8Pf3d9pnsVjo378/8+fP59ChQyQkJNR6fbt27Xj22WfrfO/CwkKAWoXPm516tq5D6z4E+ddcSvxs035mrdtLSUWV03Ep2YXNks/tVBURXnzS71n7K+H8T8EnwLxMIiIi58DUYpeW5ljHNC4urs79sbGx1cfVVexOp7S0lM8//xx/f3/GjRt37kE9QElFAQdztzuNnbzSxIZDOVw/66fmjuXe/MLYnfAa/Y8+AK36wPkfgdXP7FQiIiJnzfQzdgBBQUF17j8xXlRUx+z/p2G323niiSfYt28fDz/8MG3btnXpddu2bXP5a5yt9evXN9l7Z1XuwDDs1dsBljAOpeSSbjkGwFcpx1x+r4KsDNavL230jG7JN5rNMS9T5RMGG7eYnUaOa8q/K3L29Lm4H30m7snMz8X0hydcYbFYXDqurKyMRx55hEWLFnHNNddw1113ufw1+vTpQ0BA012CW79+PYMGDWqS97Ybdr5ctwBOusLaP2kMvdrV3F+3pjwZ1mac8b16tAnnvstGERXshZcj7VWQswZiRlYPrV+/nv7DJpgYSk7VlH9X5Ozpc3E/+kzcU1N/LuXl5fWejDK12J24/+3EmbtTnRgPDT3zlBO5ubncfffdbNq0id/+9rc8+OCDLhdCT5d+LJni8vzqbR+rH53bDKz3NZf3as+Do53Xjg328+G8+Cj8fb1w6TFbBfxyAxyaA6M+h4QpZicSERFpdKYWu4SEBCwWCxkZdZ9JSk9PB6BDhw71vk9OTg433HAD6enp/POf/2TKlCmNHdWtnfrQRKeY/gT41n15+4T4VsGM7RLblLHch60cfr4G0uc6tldcC5esh4i+5uYSERFpZKYWu6CgIHr27MnOnTspKytzWiXCZrOxadMm4uPjadeu3Wnfo6ioiDvuuIPMzEzefPNNRo4cedpjvVFhWS6HjiU7jZ380ESLV1UKP02FjHk1Y13vdTwsISIi4mVMn8fuqquuoqysjE8++cRpfM6cOeTm5jJ16tTqsdTU1OonaU/429/+xq5du/jXv/7V4kodQErmWqBmnr/okHiiQ9ubF8idVJXA8snOpa7XozDwX9BCLtOLiEjLYvrDE9OmTePbb7/lueeeIz09nb59+5KSksI777xDjx49uP3226uPnTRpEklJScyb5/hBvWvXLmbPnk23bt2orKysHj9ZVFQUQ4cObbZfT3Oy2avYfWSt01j3uGEt5t7CelUWwY+XQ9aPNWN9/gR9n1KpExERr2V6sfP39+ftt9/m1VdfZd68eXz88cdER0czbdo07r//foKDT78CwI4dOzAMg+TkZB544IE6jxk6dCizZs1qqvimSsvdQVllzVQwfj4BJMX0NzGRm6jIh2WTIPuXmrF+z0CfP5qXSUREpBmYXuzA8dTro48+yqOPPlrvccnJzveSTZ061elSbUuzK2OV03bnNgPxa+mrJlQcgyUXQ+5JZzIHPAe9fm9eJhERkWbiFsVOGi6vJIvM/L1OY3poAshc4lzqBv4betR9NldERMTbmP7whJydfUc3OW23Ce9IZEgLmb6kPom/gsGvABYY8rpKnYiItCg6Y+ehisqclwjrpHvranS7F9qMgYjeZicRERFpVjpj5yV8rf5mRzBHyWGoLKg9rlInIiItkIqdeK7iA7DoAlh6qWN6ExERkRZOxU48U9FeWHih4/+zf3GsLmEYZ36diIiIF9M9duJ5CnbD4nFQ6lhLGKs/dL9fEw+LiEiLp2InniV/ByweD2WZjm2fQLjga2h3samxRERE3IGKnXiOvK2OUld+1LHtEwyj50LsOHNziYiIuAkVO/EMuRth6UVQnuPY9g2FMd9DmwvMzSUiIuJGVOzE/WWvgaUXQ2WeY9svHMbMg5gRpsYSERFxNyp24t4MAzY9elKpi4BxCyB6iJmpRERE3JKmOxH3ZrHAqM8hoi8ERMOEpSp1IiIip6EzduL+AlvDuEVQdlQrSoiIiNRDZ+zE/ZRm1h4LbKNSJyIicgYqduJeDn0D33SC/Z+YnURERMTjqNiJ+zj4Jfz0K7CVwsqb4NBcsxOJiIh4FBU7cQ/7P4YV14FR5dgO6QiR/U2NJCIi4mn08ISX+e+qFP61bAe5JeVO46WVNpMSuWDv+7D6NjDsju3w7jBuMQTHm5tLRETEw6jYeZG0Y8X89vNVZsdomNT/weo7AcOx3aqXo9QFxZoaS0RExBPpUqwXST5a4PKxCRHBTZjERbtfg9W/prrURfSD8ctU6kRERM6Sil0LNLxDa349vKu5IXb9G9bdW7MdORDGL4HAGNMiiYiIeDpdivViF3Rqw+e3jHYa8/OxEhHkb1Ki4/J3wsZHarajh8HYeeAfYVokERERb6Azdl7M38dKTGig0/9ML3UArXrC0P86/jtmlGPtV5U6ERGRc6YzdmKOzrc71n5tOx78Qs1OIyIi4hVU7KTpGQbYSsA3xHm8/ZXm5BEREfFSuhQrTcswYMNDsGgMVOSbnUZERMSrqdhJ0zHsjidfk1+C3HWw9BKoLDI7lYiIiNfSpVhpGoYd1tzlmID4hOD24BNgXiYREREvp2Injc9ug9W3w773a8Y63AAj3gOr/siJiIg0Ff2U9UBZhaVsy8wjyKdm7KMN+9h8pNC8UCfYK2HldDjwSc1Y0i0w7H9g9Tn960REROSc6R47D2MYBpe9tYSN6blO4wuSM/h6a5pJqY6zVcCKac6lrvOdMPxtlToREZFmoGLnYTILS9lwKPfMBwJBfs14QtZWDj9fDWlf1Yx1vReGvgEW/TETERFpDvqJ62EqquwuH3vDwKQmTHKKLU9C+tya7R4Pw+CXVepERESake6x8xI3DEzimoGdAbBaYFiHGIYmtm6+AL3/D44sgdz10Otx6P83sFia7+uLiIiIip23mNg9ji5te5oXwD8Cxi6Ag59Bl9+o1ImIiJhA18nk7Nirao8FREHX36rUiYiImETFThqu4hgsHAV73zU7iYiIiJxEl2LdlGEYbEo/RmqO89x0R4vLTEp0XFk2LL0Ijm2CnDVg8YOkG83NJCIiIoCKndv664It/GXBFrNjOCvLgsXjIX/b8QEDbMWmRhIREZEaKnZu6tUVyWZHcFaa4Sh1BTuPD1gcq0l0vs3UWCIiIlJDxc5N5ZdVmh2hRskhWDwOClMc2xYrDH9fl2BFRETcjIqdh5jSNwHrSU+bdm2d6bTf0lQTARftd5S64n3Hv5APjPwIOlzbNF9PREREzpqKnYf4+KYL8Pd1rLdaWJbLl+vmOe2PDG7b+F+0MNVR6koOOratfnD+Z5AwpfG/loiIiJwzFTsPtDtzDWBUb0eHxhMV2q5xv4itDJaMP6nUBcAFX0L8ZY37dURERKTRaB47D2OzV5FyZK3TWPfYYY3/hXwCof/fHffT+QTC6G9U6kRERNyczth5mIM52ymrrJlixM8ngKSYAU3zxTreAIYNguIhdlzTfA0RERFpNCp2HiY5c7XTduc2A/Hz8W+cNzeM2suBJd3cOO8tIiIiTU6XYj1IXkkWmfl7ncYa7TJs9mpYPBbKcxrn/URERKTZqdh5kFPP1rUJ70hkSOy5v3HWz7DkIsj6EZZMhIq8c39PERERaXYqdh6iylZB6pH1TmM9Yoef+xsfWQbLLoGq42vSlhyEkvRzf18RERFpdip2HuJgzjYqbGXV2wG+IXRo3efc3jRjISybBFXHH8YIbAvjl0FE73N7XxERETGFHp7wEClHnC/Ddm07CB/rOXx86d/DT1PBXu7YDmoH45dAePdzSCkiIiJm0hk7D5DYqpSc4kNOY91ih579Gx6aAz9NqSl1wQkw4UeVOhEREQ+nM3YeYHTSMaftdhFdCQ9qfXZvdvBzWHEDGFWO7ZCOMH4phHY8p4wiIiJiPhU7Nxfoa2N4Qr7T2FlPcXJ4Pqy43jHpMEBoF8fl15CEc0wpIiIi7kCXYt3ciIR8An3t1dvB/uEkRPU8uzeLGQnRxy/hhvdwXH5VqRMREfEaKnZuzWB0Uq7TSNe2Q7Bafc7u7fzCYMwPkDTd8fRrcLtzTigiIiLuQ5di3VjnqFISWpVXb1uwnNtDEwD+rWDEe+eYTERERNyRzti5sTGnnK1rH9WTkIBWrr/Brn9DyuuNG0pERETcls7YuakQvyqGxBc4jXWPa8BDE9ufhc2PO/7b6g+d72jEdCIiIuKOdMbOTY1IPIafj1G9HRIQSXxEV9devPXpmlIHsO99sNsaOaGIiIi4G52xc0OGYeeCDqc8NNFmKBbLGXq4YcCWJ2H732rG2o6F0XPhbB+4EBEREY+hYueGMvL30ja0onq7ym6hU5uB9b/IMGDTH2Dn8zVjsRPhwtngG9xESUVERMSd6FKsG0rOWOW0vT49jEC/0NO/wDBg/YPOpa7dZTB6jkqdiIhIC6Izdm6mpLyAgzk7nMaW7Ys6/QsMO6y9B/bMrBlrPwXO/xR8/JsmpIiIiLglnbFzMylH1mJQs9LE4YIAdufUc9bt1FKXeC2M+kylTkSa3csvv0z37t2r//fiiy+aHUkaICcnh8GDBzN48GCys7PNjuPRysrKGDduHL179yYlJQWA8vJyp78f48aNa5KvrTN2zaDKVkmJPZesgoOn7DFYnnqElOzC6pGYgBX4nlS3l+2LBCynf/PYiyD1Lcf6rx1vguHvgNU7P9a0tDTeeustfvnlFzIzMwkICKBdu3ZMnDiRW2+9ldDQmsvVO3bs4M0332TDhg3k5ubi5+dH9+7dmTZtGlOmTAHgq6++4vHHHz/NV4OhQ4cya9YsAL755hveffdd9u3bR0hICAMHDuShhx4iKSkJgMLCQv7+97+zcuVKCgsL6dGjBw8//DCDBg0CoKKigjfeeIPvvvuOzMxMWrduzfjx47n//vsJDQ3lscceY/bs2XXm+Mc//sHUqVMB2LVrF0899RQbN250ylcfwzCYPXs2s2fPJjk5mZKSEqKjo+nTpw/Tp09n2LCaaXRsNhvvvPMOc+fOJT09ndLSUqKjo7nwwguZMWMGsbGxAIwbN4709PTTfs3333+fYcOGceTIEf71r3+xatUq8vPzSUhI4Nprr+Xmm2+uPnbRokW8/vrr7N27l6ioKPr27Uu/fv3w8/OrPmbhwoX87W9/IyMjgxkzZnDfffdV78vJyeGll17i559/5ujRo3Ts2JF77rmHSy+9tFauXbt2cfXVV1NZWVnrfepi5p+5d999l88++4y0tDQCAwMZMWKE0585gDVr1vCXv/yFPXv2cNVVV/Hss89W73vyySf57LPPan2Niy++mP/85z+A48/G559/zgsvvEBeXp7Tn7UTv/4XX3yRdevWkZubS8+ePXnkkUcYPnx4vb9vY8eOZfz48fTs6bz84ZIlS7j77rsBmDp1Kv/4xz+c9r/88su88sorTJo0qVYpfOihh/j++++rP7e6/s74+PgQGxvL0KFDmTFjBu3bt6/eV1JSwqxZs5g3bx779+/HZrMRHx/P+PHjueOOO4iMjHR6r02bNvH222+zYcMG8vLyiIiIoEuXLtxwww1MnDix3l//qU79Mz5hwgR+97vfOf0ZP2H16tVMnz69zvc58eejqqqKN954gy+++IJjx44RFhZW63vSli1b+Pe//8327dux2+1069aNu+++m1GjRp0257///W8KCwt57LHHaN26NQCfffYZs2bN4tChQ7Rt25YpU6bw29/+ts7X1/fn+8SfzzN9LzxbZ/oeXJcPPviATz/9lEOHDhEYGEivXr144IEH6NevH+D4M/Dyyy+zdetWSkpK6Nq1K3fccQeXX345AEVFRbz++ussXLiQjIwM4uLiuOWWW+jRoweBgYE8+uij3H///Tz77LP873//w9fXl2eeeQaAJ5544qx/rWfinQ3AjeQUHWb+1jepsJWRumVxncfEBdX92vIqCyvTIur/Aom/AvsHcGQxDHnDa59+3bBhA7/+9a8pLi5mzJgxXHvttVRVVfHtt9/y8ssv88MPP/Dpp58SGhpKamoqN954I+Xl5dxwww0kJSVx6NAhPvjgAx599FGA6h+0AAkJCfzmN7+p9TVjYmIAWLBgAb///e9p27YtDzzwALt37+bLL79k27ZtzJs3D39/f/74xz8yf/58pkyZQvv27XnjjTf4zW9+w/z584mOjua5555j1qxZDBw4kJtuuon58+fz3nvvkZ2dzb/+9S9+9atf1foG9N5775GSkkJCgmM931dffZXXXnuN8PBwl3/fDMPggQceYP78+bRu3ZqpU6fSrl07du7cydy5c1myZAlPP/0011xzDQB/+tOf+OKLL+jevTt33303vr6+fPfdd3z++eesXbuWH374Aau15l8et9xyC1271p6GJykpCbvdzl133cWuXbuYMmUKPXv25L333uOZZ57Bz8+PadOmsWvXLu6//35iYmK49957Wbx4MT/88AMJCQk88sgjVFVVcc899/Djjz/W+sF74td37733snHjRi644AJuvPFG3nvvPR566CEiIiIYMWJE9bFVVVU8/vjjGIZR633qYuafuTfeeIMXX3yR3r1784c//IFVq1Yxf/58tm3bxqJFi7BardXFLSqq7ls18vPzAZg+fTrdunWrHj9RdvLy8vjNb37D5s2badWq9sTnpaWl3H777Rw8eJDLL7+c4OBgvv/+e+68807mzJlDp06dTvt717t37+o/Uyc7UcRCQkKYN28eTz75JMHB53Yf8IUXXlhdtMrLy1myZAmzZ89m3bp1zJkzh5CQEAoLC7nxxhtJTk6me/fu3HbbbQQGBvLjjz/y3//+l2+//ZYPP/yQ+Ph4wPEPucceewyAyZMn06NHD7Kysvjyyy+57777uP/++7n33ntdylfXn/F3330Xf39/HnnkkVrHJyUlVf/wP2Hr1q18+umn1Z/d66+/ziuvvEJsbCwPP/wwmzdv5rvvvmPbtm0sXryYw4cPc+utt1JRUcGdd96Jj48PM2fO5Le//S1ffvkl3bt3r/V18/Lyqn+/rr32WgCWLVvGk08+SdeuXbnvvvv46quvePHFF4mIiGDatGm13mPgwIG1si9fvpwFCxZUZz/T98Kzdabvwaf66quv+Otf/0rr1q257777SEtL46OPPmLLli0sWrSIY8eOceutt2KxWLjjDsc8sG+++SaPPPII0dHRjBgxgj//+c98++239O/fn+uvv54vvviCp59+mt/85jcMGjSIiRMnkpCQwM8//8yePXvo0qVL9d8LFTsPtivjFypsZWf12rXprSip9MHPx4rVUs9Zu47THP/zUna7nccff5zi4mLuu+8+ZsyYUb3v9ttv5+abb8bPz4/U1FT69+/PkiVLKCkpYdSoUU5/eXr06MHWrVtr/aswKiqqzh9CJ2zfvp2BAwdyxx13MGHCBAAWL15Meno6u3btol27dixcuJAOHTrwz3/+E4CMjAy+/PJL5s6dy6233srBgwcZOHAgzz//PPHx8QwdOpTJkyezdOlSDMNgyJAhDBkypPprrly5kj179nDxxRdXj69fv57HHnuMLl26cOutt7r0ezdnzpzqUjdnzpzqf4UDjB49mmeeeYY9e/ZUj3377bcAPP300wwYMACAa665hr/+9a/ExsaSn5/vVLBGjRrFhRdeWOfXTktLIzg4mHHjxvHss89isVjw8fHhmWeeYcmSJUybNo3PPvsMm83Gww8/zJVXXslll13GmDFj+Oyzz3jooYcoLy/n4MGDvPXWW2zatIlXXnml1tfYuHEjfn5+vPzyywQFBZGYmMiMGTOYNWuWU7F744032L9/P1deeSVffvllvb9vZv+Z8/PzY/r06dxyyy20b9+eq666ikGDBpGens7Ro0f/f3t3HldT/j9w/HXbV6UoJLK1IAZpGGNfIltm7Ma+M3ZmpmEsYx07YTLGUrJkC4WyL40SkW2EsgxZilTS3r2/P/rdM13diJBv83k+Hh4PnfVzzufcc97nsx0sLS25cuUKixcvJjMzU20pSUJCAgAdOnTAwcGBrKwslSAqNjYWuVzOjh072Lp1a57Sr4iICP755x/KlCnD4sWLuXjxIlWqVGH+/Pls27aNqVOnvvEcvu7FixecOHGCsmXL0rp1a7y9vQkKCqJLly7vtJ3X2dnZqZzLnj170qxZMx48eEBERASNGjVi2bJl3Lx5E2dnZ9avX4+OTk5TlaFDh/LTTz+xd+9epk+fzvr160lOTmbWrFlkZ2ezdOlS2rdvL227R48edOvWjYsXL5KRkcH06dPzLWm3srLi+PHjb73Gc78oAVhYWKgcT1paGn/++SdGRkZMmDAByHnpAGjXrh39+/cnLS2NAwcOEBMTQ2xsLNeuXcPOzo6GDRsyduxYAG7evMnhw4c5c+aM2sAuMDCQ9PR02rdvj6GhIQDbtm0DYObMmTg5OVGrVi369OnDtm3b1AZ2NjY22NjYSH8/f/6cJUuWYGVlJQVHb7sXyt70rMvHs2fP3noPfp3yHLq5uTFo0CAAgoKCeP78OVFRUcjlcrp160adOnVwdXUFcoL0I0eOcPnyZerVq0dgYCAAixYtomLFinz55Zd06dKFwMBAJk6ciEwmo3PnzqxatYr9+/czceLEdz629yECu48sLTPlvdbLksPR6Jw38c41rdHS1IDsNLj0A9ScBnoW752mE1FP+H73OSJjk96+8Edgb1GCVd9+SfOqZQq0/PXr17l37x76+voMGTJEZZ6uri6+vr4qNwNl8BIeHo63tzetW7embNmydO7cmc6dO+fZflZWFnFxcXmmm5iYoKOjI91MlR48eEBycjIAOjo63LhxA7lcrlJypSwhuXr1KpDzppfbjRs3gJwH+Os3srS0NKZOnYquri4///yzNN3T0xMdHR3OnTun7jSpdejQIQB69eqlEtQBtG3bFhcXF5X9m5ubExMTw4oVKxg1ahS1a9fGwMAgT5WZUlJSUp5zp6WlRcmSJbG2tpYeDEp//53TMUj5YL1+/TqA9KApW7YsBgYGJCQk8M8//1ChQgX8/f3R1tYmIiIiz/5fvXoF5JxHPT09AKnU5cqVK9JyN2/exNPTk8mTJ/Py5cs823ldUV9zyodgamoqjx49Yt++fQA4OTlhYZHz29+5cyc6Ojrs2bNH7TEoS+x+/fVX6Rpt1KgRs2fPply5ctjY2LBjxw5kMhlbt27Ns77y3BoZGUnHqu7cFpS/vz+ZmZm0bduWtm3b4u3tzZ49ewod2L1OS0sLQ0ND4uLiSE9PRy6XSy8so0aNkq49AJlMxrhx49i7d69UlR8eHk5ycjLW1tYqQR1AxYoVCQkJQVMzp2ZEXUm7kjI4UneNGxsbS9d47kBIHQ8PD+7du8dPP/0k5X3Tpk05e/asFHyfP38eAAcHBywsLHBxccHFxUXaRkZGhvQCl/v4c1MGOrmP5/W0K+9rt27dIj09HV1d3Temffbs2bx48YJff/0Vff2cqql3uRcWVEHuwa9r0qQJO3fu5Ny5c9y5c4cHDx6QkJCAhYUF9vb2GBoaUr9+fRQKBbGxsdy9e5eLFy+ir69Ps2bNSE1NJSsrCwBjY2Pg39/HP//8Q1paGnp6ejg5OQH/nt9PQQR2n1gJvVLoaOdc4FcevSA1898vQlga6aGtqYFcocPLrKoM/LI8NmZGdP+iImSlwOku8OQwxJ6EFsdBr1Q+e3mzkTtDVdr1fWqRsUmM3BlKpLtbgZZ/8OABkPOjUT68IadEIjMzU/pbW1sbU1NTXF1d2b17N+fPn2fu3LnMnTuX0qVL8+WXX+Lm5kbjxo1Vtn/9+nW17U5Wr14tldApPX/+nFGjRpGVlcXXX3+Nvb291DBWeSMHpBIaZalJbpcvX+bXX38FkN4Uc9uwYQMxMTEMGjRIatMG+d+Q3+Sff3LadVapUkWalpGRIT30lZQBhbu7O5MmTeLs2bOcPXsWbW1tatSoQZMmTejVq1eeaj91VUn29vZSIJLbli1b2LNnDxoaGtIbtPL85D53+vr6pKSk8OLFC2xsbPKUaORWqVIlTExMSExM5M8//6RNmzbSg+PFixfAv1Ww9vb29O3blzVr1uS7PaXP5Zrz9fVl/vz5yGQy3NzcmDVrlvTwe9v1oMxjS0tLBg0axL59+zh16hQTJkxg+/btb12/evXqaGtrExUVxb59+5DJZOzcuRP499y+C2XJVvv27XF0dKR8+fKcP3+ehw8fqrSFe1epqalSkJyRkSG1oTM1NaV+/fq8ePFCOhe5q6SVypUrh7GxMS9fvuTevXvSbya/qmZlUAfkKWlXR901bmRkxMuXL6VrPD8PHjzAy8sLS0tLevfuLU0fMGAASUlJrFmzhtatWwNQs2ZNfv/99zzBkfL6v3PnDqVLl6Zjx45q93X//n1A9V7xetqV9zW5XE5iYqIUaKpz4cIFDh06RI0aNfJtk/i2e2FB5XeOc897XZs2bfj111+ZO3eu1B7XxsaGNWvWqGzn5cuX0u+3WrVqeHp6Ym9vD+Tcf+7evcuaNWvo168fPj4+QE4TkcTERPT09KTzee/evfc+vnclArtPrF6ldlQ0rwGAe9B+/n767wP2ypSO1ChjmnelrFdwqiM8PZHzd8JV+GcH2I76BCn+fCjfjpRGjhyp8hakbFisq6vL5s2bCQ4O5tSpU0RERBAZGUlAQAABAQF5qtYqV66stiqrRo0aKn9HR0czbNgwHj58SPXq1Vm4cGG+ac2vHdeRI0eYMmUKqampuLm5MXToUJX5CQkJrFu3Dh0dnTzz3iYuLo7s7JwXBU1NTam9FiBNh5wb7sCBA1XWVXZ2aN26NUeOHOHQoUOcP3+ey5cvExERQUREBNu2bWPv3r0qJX/jx4/Pc55er3ZUKBQsX74cT09PNDQ0mDFjhvQWq47y3BXk7V1PT4958+bxww8/sHjxYhYvXsxXX30lnQPIKSG4desWu3btUnkoK6WmpqoEurnbmxX1Nde6dWvKly/P8ePH2b17N48ePWLjxo1oab391h0QEIBcLsfY2BiZTEbTpk1p0KABERER3L9//60lReXKlePnn3+Wzi8gVW2rO49vEhkZyd9//03FihVxdHQEcqqIPT092bt3r8q5eVc+Pj7SA1XJ0dGRmTNnYmxszPPnz6XpuX8HucnlOSMRyGQy6brLb9ncXr58SVqa+qY2mpqa+bZ/LOg1vnz5cjIzMxkyZIhK6Zivry+///471apVY9SoUURGRuLp6cmIESNUgvbk5GTGjRtHcHAwpqamrF69Wm1bVfj3ReBNbXhz39felvbFi3PGVc2vLeLb7oVKycnJUg0J5DRjKMhL7tva0p48eZI5c+ZQvnx5vv/+ex4/fszy5csZOnQou3fvls6TgYEBf/zxB/fv38fDw4PBgwezYcMGHB0dmTdvHmPGjGHz5s1s3ryZmjVrSi8Jyt+I8n6SlPTpashEYPe5y3wJJ9tD3Jl/pzn+Wqig7vduDRizJ4wbTxPfvvBH4GBpgsc3zgVeXtnLKyYmhqSkJOnG89NPP5GYmMjNmzelm4iSTCajcePG0ptWUlISixcvxtfXl02bNqk8SIyNjfNtJ6Z0/fp16S3Z1dWVOXPmSG91ykBHWXUFSNV9uYOgnTt3Mn36dGQyGZMmTWLYsGF59nPgwAFSUlJo3rx5vg+F/PTo0UPqqaps31OpUiXu3LnDjRs3pJ5cDg4OrFu3DshpG3Lr1i2V7VhaWjJgwACpVO3ChQuMGTOGuLg4AgICVNqrKEvz8qNQKHB3d8fPzw9TU1MWLVqksry5uTn37t1TOXcpKSnSvIJo1aoVp0+f5ubNmxgbG2NkZETz5s2xtLQkPj6eNWvWYGVlxaFDhzh06BAXLlwAIDQ0FENDQ0xNTVWCrPnz50u9OYvymktJScHKygorKytatWpFaGgoYWFhnD179q3rQt4A28jICBMTE549e8azZ8/eGtgB9O7dm3bt2hEVFUVsbCyWlpaEhIRgaWn51nVzU1YX379/P0/7Lj8/P0aPHo1MJpOCF+U1kJvyN6Ws0lNq166d1JN39erVRERE0KRJE2rWrAnkBAJmZmbEx8dz48aNPKVMDx484NWrV2hoaFC5cmUpwImOjkYul+cpMb558yaVK1dGW1ubuXPnvrWNnbprXPmQf9M1npiYyOHDh9HU1JR+u0oeHh7I5XIpf9q1a8fJkye5fv06Z86coWXLliQmJtKvXz8iIyOxt7dn5cqVVKxYMd/9Kb3eNOPJkye8evUKY2NjKd1aWlpqO9woRUVFcenSJUxNTWnatGme+QW5Fypt3LhRpW2t8iU0t4Leg3Pz9PQkIyOD0aNHS1XuFy9e5NixY/j7+9OvXz+ysrKQy+XSMSQlJeHh4cGGDRtYtmwZdevW5ejRo9y6dQtNTU3s7OyoU6cO2tra+QbQn4II7D5nGQlwoh08z/UlitrzocZPhdps86pluPZDp8Kl7ROyt7enatWqREVFsWbNGqm3Wu3atYG8pSrz5s3j/PnzTJ48mUaNGgE5b6Fdu3bF19eXjIyMAveMBHj69ClDhgwhKSkpT8kL5ARKWlpaKgGSst1InTp1gJyeYdOnT0dHRwcPD498H8z+/v5ATvu3d7VgwQKp9EBZfdihQweOHTvGzp076dOnD+XKlaNkyZI0adIEhULBihUrpPWVXft1dXVZvXq1dIN3cnLC0dGRU6dOkZGRkXfHb7Bs2TL8/PyoWLEi69evl3r4KtWqVYvw8HBu3ryJvb291DaldOnSBaqeUygU7N+/n/v37zNixAh0dHSkIT6cnZ1JSUkhMzOTe/fu4enpqbLuhQsXePz4Mdu2bZMCXchpT2RhYVFk11xWVhbNmjUjLi6OAwcOULVqVRQKhbT+m6qmlV68eMGkSZNISUnBy8sLXV1dnj59Snx8PDKZ7I09WpXS0tIICAjg2bNnjBgxgvDwcE6fPg3knNuCyszMlNq4ubq6qnTgOH78OA8fPuT8+fM4OzurtItKTk6WgtP4+HipXZ8yYFMqX7689HsqV64cbm5urF27lubNm+Po6Cg1YN+4cSNr1qzB2dlZCg4VCgUeHh4A0stUo0aNMDc3lxrf5+7IcPfuXXr27ImBgQH+/v4MHDhQalj/OuVvUN01npKSIl3jmZmZKtW/yt9dUFAQGRkZNGjQIM9LnrI0Mfd1qPzty+Vy5HI5I0eOJDIykubNm7Ns2bI8AfHr1JUs1apViydPnnDz5k2cnJyIjIyU8kBHR0dqA6qlpaUSNCrvYy1btsxTulzQe6FSp06dpN8doLbjR0HuwbGxsbx8+ZKSJUtiZmYmnbvc97T09HQg5xwuWLAALy8vBg4cKJVYK9dR/gaPHj1KZGQkPXr0oHTp0vz1119kZmZSo0YNqcSuICWhH5oI7D5X6fFwog3Eh/87re5SsJ+Q/zrFlEwmY8GCBQwcOJCNGzcSHR1NkyZNkMvlXLlyhSNHjgD//uCtrKzw8vJi3LhxdO3aFWtra5KSkti7dy+QM35W7rfS+Ph4qe3Q6zp37sz8+fOJj4/HysoKS0tLlWXr16+PjY0NHTt2xM/Pjx9//BErKysOHDhAqVKlaN++Penp6cycORO5XI6TkxNPnz5V2YarqyuGhoZkZmZy7do1AGkcpdw2bNhAYmIijx49AuDhw4fSWF+DBw9W+7B1dXXl+PHj+Pv706VLF3r16oWlpSWxsbHSm2bJkiWxtLTE3Nycu3fvEhMTQ79+/WjWrBm6urpcv36d06dPY2BgkCfgDA4O5unTp3n2W61aNfT19fnzzz+BnIbKoaGhhIbmvKQYGhri6upKjx498PHxYenSpTx79ozDhw8DOUN0yGQykpKSWL9+PYBKSVtWVhYmJiYMGjQIf39/zpw5Q1RUFNWrV2fdunXo6ekxePBgypcvz82bN1XSphwrLfc4dupKoIrymnN1dcXLy4vx48fTtWtXrl69yqNHjyhXrpzUW1mZ98rj+/vvv6VpEyZMIDs7m0uXLjFy5EiaNGnC/v37kcvldO/eHTMzMx48eMCuXbukdSFnaJ/79+9Tvnx5unbtysaNG4mKiuLRo0ekp6dz4MABzM3N1faIzM+pU6d4/vw5NjY2ecamW7p0KWvXrsXPzw9nZ2eaNm1KkyZNOH36NL1796ZTp07I5XJ27dpFYmIiLVu2fOMYelWrVmXEiBF4eHjw008/sWfPHnR1dRkzZgxhYWFERETQrVs33Nzc0NHR4dixY4SGhlKhQgVmzpwJ5ARk8+fP5/vvv+eXX37h0qVL1KhRg0ePHrFz505SUlIYMmSIVBKoLtDI7W3X+NOnT6Xg8MqVK1Kp5aVLlwD19wJXV1d8fHzw8vJCLpdz+/Zt7t27R6lSpahfvz7bt28nPDwcbW1tGjRoIAXWkNOOTF27wIoVKxIREUF0dLR0jfXp04fDhw8zc+ZMvvnmG+ma7d+/v5Tefv36UapUKf766y9pW8rmCq+nvaD3wtfT9baSRlNT0zfegyHnWvPz82Po0KFMnjwZV1dXrl69ysqVK0lMTCQ2NpaQkBCpc0RycjKbN2/Gx8cHbW1tqbkFILXJO3PmDNu3bycsLIwmTZrg7e2NpqamSoeg6Oho6bx/KiKw+xylPYPjrSDh8r/TnFaBbcHGTSqOHB0d8fPzY926dfz111+EhoaiqamJhYUFrq6udOnSRSqe79+/PxYWFvj6+uLv709CQgI6OjpUqlQJd3d3lcFxIacqJr8xhVxcXKSSgpiYmDzLzZ8/HxsbG2bMmIG2tjZnzpwhOTkZZ2dn3N3dMTY25uHDh1IVaXBwMMHBwSrbaNiwIYaGhioN83O3j1Py8fFRGRT40aNHUilUt27d8n0jXLRoEV999RV79uzBx8eH1NRUjI2NqVq1KpMnT6Zbt26YmpoCOW13/vjjD06fPs2qVavIyMjA3NwcV1dXRo4cSYUKFVS27eXlpXaf/fr1w8HBQSpZeH0gZSsrK1xdXalUqRK///47K1euZNWqVZQuXZpvvvlGam+TlJSktqTtwoULWFlZMWjQIBYuXMisWbMICQnh5MmT1KxZkylTphSoVOpNivKamzJlCqVKlWLHjh0sXrwYU1NTXFxcmDBhglSK9fp5uXnzphTkTZgwgRUrVrBy5UpOnjxJWFgYpUuXZvz48VIv39zXj9KJEyc4ceIEzs7OdOvWjTVr1vDrr78SEBBAVlYWDRo0wN3d/Z2aCSirYXMPfKz0zTffsHbtWpUx7VasWIGXlxcHDhxg9erV6OjoULFiRYYOHVqgHrTDhg0jMDCQ27dvs3z5cn788UcMDQ3ZunUr3t7eHDx4kFWrViGXy7G2tmbkyJEMHjxY6tkIOb1Od+3axbp16wgODmb//v3o6Ojg6OhI//793+mLAequ8e+///6tbWhjY2MB9VWJP/zwAyVLlmT37t0sWbIEHR0dWrZsKY3fqLxnZWZm5unR3qVLF7WBXd26ddm3bx/h4eF8++23ADRo0ICFCxeyYcMGPDw8KFeuHNOnT8+3lFJJ2Znl9bTHxcUV6F74Pt50D1ZnwIABaGtrs2PHDmnAbmdnZ8aOHSsFYZs2bZKuRw0NDWxsbBg0aJDUwcnd3R25XM7Ro0e5ePEi1apVY9asWSr34vDwnMIZZcnhpyBTvEudVDGUnp7OtWvXqFmz5lu7br+PY3978yD+32+/NnfoK3WecFyopvOEaSYcawGJ1/5/qgyc10LVd2tILxRMeHj4G0cmFz49kSefp4Lki7rSUOHj+ZC/lfj4eJo2bYqOjg5nzpwp9MDR/2XKfFEoFLi4uHD//n38/f1VemXb2dlJ7TDf1dviFvGt2M+NliHoKt9yZDmfCBNBnSAI/0OuX7/Ozp07paYFwufPzMyMzp07k5ycrPZTdMK7O3r0KPfv3+err77C1taW7Oxsdu7cmW8zjA9FBHafGy0DaOoPFk3hKx+o3L+oUyQIgvBOTpw4wbRp06S2iML/hvHjx2NsbMyqVatUhokR3l16ejoLFixAS0tL6nWflZXFtGnTPurnxEC0sfs8aRtBy+MgE3G3IAj/O8aMGSOqYP+HlSpVSuqkJBSOrq4ux44dyzPt9c5cH4OIHIpYFd2nDC11Iu8MEdQJgiAIgvCORIldEaqk9ZDfbedhpfMCbVkWoP5TL4IgCIIgCAUhioU+saxsOelZ2WQ8v8yG0lOx0sn55uICK1+00p8UceoEQRAEQfhf9lmU2Ck/03Hs2DFiY2Olz5CMHz9e7Xher4uIiJA+JZOenk7FihXp0aMHvXv3LtAo7R9TWpbq9wZ7eJ8mO2E/h6v9RmntnE+eJGfr0ilqIh6ty6jbhCAIgiAIQoEUeWCXkpLCd999R3R0NH369KFmzZrcu3ePDRs2EBoayq5du974zbWQkBCGDh1KmTJlGD16NKamphw+fJjZs2dz7969j9775G2i4pIwyTXMjIP2fVbYrsFMK+ebdknZerS/PZmzr2zz2YIgCIIgCELBFHlgt3nzZm7evMmMGTPo3bu3NN3BwYHvv/+etWvXSt9pfJ1CoWDWrFno6emxdetW6ePObm5ujBw5Eh8fH7p27Yq9vf0nORZ1UjKzpcCudHoMv5vvwkgj55t+CVkGtLs9hbCUKmhpyKhg+n4jbguCIAiCIMBn0MbO398fAwMDunbtqjK9VatWlClTBn9//3w/nn3t2jXu3r1Lu3btpKBOqW/fvtIHwj8HFmkPaBO7QwrqnmcZ0S7anYj0alQoacgf3RtirKddxKkU/ksePnyInZ2d9O9tH+MWPj8TJkzAzs6O7du3F3VSir1bt25Ro0YNWrZsKX0s/vTp0yq/ofwKIQThUyrSErtXr15x+/Zt6tWrh46Ojso8mUxG7dq1CQoK4uHDh1hbW+dZ//LlnG+pqvtIcu3atVWWKUpl0u7TMm432oqc74CiWxrzdkcJKZk33UL+Hjx4wJ9//snZs2d58uQJurq6lCtXjjZt2jBgwADpG5qQ81HzP/74g4sXLxIfH4+2tjZ2dnb07NkTNzc3IOcblsqBI9VxdnaWvnGamZnJhg0b+P3330lNTcXb21v6TijAy5cvmTdvHiEhIbx8+RJ7e3smTpyo8rmfkydPsmLFCqKiojAyMqJZs2ZMnTpVSndQUBDr168nKioKLS0tmjVrhru7OyVLluTcuXP069fvjels0aKFyrdkczt27Bjly5dXO8/U1JTJkyejr6+vMj0rK4umTZvy7NkzDAwM+Ouvv1Q+M/Tw4UNatmwJqH68HHIeeEOHDpU+mZNf+kuWLEmVKlUYNGiQtC2lU6dOsXXrVi5fvkxycjImJibUq1ePgQMH5vnu4osXL1i/fj3Hjx8nJiYGLS0t6doYOHCgyrXxNk+fPmXOnDlcunSJjIwMHB0d+fnnn6lSpUq+60RERLBw4UKuXbuGnp4ezs7OTJ8+XXrhfPnyJYsWLeLw4cOkpKRQuXJlvv/+e+mbk5BzPa5bt44HDx5QsmRJ2rdvz6RJk9DWVv/Cd/HiRQ4ePIi9vT3du3cH4Pbt2yxYsIBr166hpaVF3bp1mTZtGpaWlmq38csvv6j9yoCLi4v0/cwDBw6wceNGoqKi0NXVpWrVqowYMYLGjRtLy967d0/ttr/77rt8z9nbvOux2NnZ5butmzdvcvHiRXr16qV2/sWLFzE0NOTp06csXbqU0NBQEhMTsba2pnv37vTt2xdbW1t69uyJj48PmzZtYvjw4dja2jJnzhzu3LnDhg0b3vtYBeFDKtLA7sGDBwCULVtW7fwyZcpIy6kL7N60vqGhISVKlJCWKUqaiiw0FDmdKFI1zdBvdRJMqhdtov7HXLx4kSFDhvDq1SuaNWtG9+7dycrKIiAgAA8PDw4dOoSvry9GRkZSe8309HR69+5NpUqVePjwIT4+Pvz4448AUnAHYG1tzfDhw/PsU9lx559//mHEiBH8888/6OnpqU3f1KlTCQoKws3NjfLly+Pp6cnw4cMJCgrC3NycsLAwRo0ahY2NDRMmTOD06dPs2bMHXV1dZs6cyZkzZxg7diyGhoYMHTqUO3fusG/fPh4/fszmzZupVKkSc+bMUdnn1atX8fX1lQK2yZMn8+rVK2l+ZmYmCxcuREtLC1NT03zPraGhId26dcsz/fTp0zx79gxDQ0NevXpFUFBQgT7C/iZGRkZSqYZCoeDOnTts2bKFUaNGsXbtWpo1awbAypUrWb16NQYGBnzzzTdYW1sTFRXF3r17OXLkCLNmzZKCmbi4OHr06EFMTAy1a9dm+PDhaGtrc/ToUVatWsXhw4fx9fUt8LcvR48ezdWrV+nfvz8ymYxNmzYxdOhQAgMD87yAAty9e5cBAwZQokQJxo4dy5UrVwgKCiIjI4M//vgDgLFjxxISEkLv3r0pVaoUa9euZeLEiQQGBlKuXDn279+Pu7s7derUoXv37vj7+7Nx40bMzc3z/Vj8pk2bgJyPmWtoaJCWlsbQoUOJi4tjxIgRPHz4kL179xIbG4uvr6/abSQm5nyrul+/firfsVReU6GhoUycOBEDAwM6duyImZkZf/zxByNHjmT//v1UrlyZhIQE4N+PsCsV5qPn73Msr/8+EhMTWbJkCeXKlVM5VkdHR3r06KGyrK6uLnK5nGHDhhEZGYmbmxsODg54eXkxZ84ctLW16dmzJwMGDGDLli34+PgwePBgypQpQ7du3Th37pwI7ITPRpGX2AF5SgqUlNOTk5Pfe/381n3dx/qmYUa6NjHmVThZyg3nF8c4abqAclGpQPhH2V9xJJfLpaDl22+/5dtvv5XmffHFF8yZMwctLS0OHDhA1apV8ff3JyUlhVq1atGuXTsA7O3t0dPTIzo6mqdPnxIeHi6VMujq6lK5cmW1+w4PD+fixYsYGRkxf/581q9fz40bN7h16xZaWjk/n8TERA4fPoylpaUUbFy7do2TJ0/y+++/065dOxYuXEh2djajR4/GzMwMW1tbhg8fjo6ODuHh4dJDunXr1jg7O+Ps7ExERARhYWHs3r0bGxsblTRmZGSwevVq9PX1admyJeHh4XlKMnbu3ElqaioDBgxQO9p5XFwckPPpm/DwvNej8kHl5ubGli1b8Pb2pkKFCnnWh5zAO3fQExUVpbLtW7duAaCpqalyHFWqVCE2NpYDBw7g5eWFsbExd+7cYfXq1WhpaTFjxgzppc7R0ZFq1aoxb948Zs6cSYkSJShdujSenp7ExMRQp04dJk2aJPWE/+KLL1i0aBGxsbHs3r0byPvwz23atGno6elx9epVatSogYuLCwDnz5/n+vXrbNiwgfr16+dZ788//yQ1NZXx48fj4OCAg4MDPXr0QE9PTzr2s2fP0qhRI1xdXUlNTWXNmjXo6ury+PFjHj9+zNKlS9HV1WX06NFoampia2uLlpYWWlpaavMmLS2No0ePoqOjQ5kyZQgPD+fcuXM8fvyYZs2a8dVXX6FQKAgJCSEiIkK6hl6nfPGtWrUqFSpUICsrS3p5CQ8P58CBA0BOrYjyZejIkSNER0fj7+9Po0aNSEpKkkrE09LSpHQnJiaqTXtBvM+xvP4bXrNmDXK5nF69ehEeHk5ERAQAFhYWVK5cmVevXmFgYIBMJuPy5cs8ffoUuVxOvXr16NatGzKZjNatW+Pl5YWfnx/VqlUDcu4lN27cYPPmzVJtkfL6fv78+Xsf8/v61PsTCqYo86XIO08UhEwme6/1FApFgdetWbOmSlXSh6JfzoIDVzYiN6hGhKIFE1oPLvIhWB4nRBMavZfE1Li3L/wRmOiXpkEVN8qa5l+1ldvVq1d58uQJ+vr6TJ8+PU+pWUBAgEo+//PPP2zbto3bt29z/fp1WrduTdmyZVWqRQHu378P5AR2uQMWKZ0mJujo6ODo6CiVmiirrWxtbaXtnTlzBoVCgaOjozStYcOGnDx5khcvXvDFF19w69YtSpQowcWLF9m+fTsKhYKGDRvy22+/UapUKenaq1atmrSNqlWr8s8//5CZmZkn7YsWLeLx48f89NNPKtV5SpGRkfj7++Pg4MCPP/6o9pp7+PChdPyvb//FixdERERgbm7O1KlTOXv2LDdu3MDS0lIqzVGuD1C3bl2V34/ypUu57aysLAC0tbXz7OvChQscOHAAIyMj6tWrh7e3NwDt27dXKVkFqFevHseOHePcuXP8888/tGnThosXLwI5JZZOTk4qy2/btg1NTU0AYmNj8y1xBWjatCnHjx8HwMnJSUpnvXr1uH79OqmpqXnSDvDzzz8DOQ/1oUOHkpGRQa1atVi4cCE2NjacP38eAEtLS3744QdiYmIoWbIkP/zwA9988w1Pnz7l8ePHVKlShV27dhEUFISWlhYuLi78+uuvGBrm7VQVEhJCdnY2derUoWHDhkBOVT/AV199JaWzZs2aPH36lOzsbLVpl8vlQM51fePGDeRyOY0aNWL27NmUK1cObW1t9uzZw4MHD7h//z6VKlUiLi4OAwMDunfvjqGhIXK5HB0dHWbOnMmtW7fQ1tamR48eTJky5Y3n+03e51hyO3XqFMHBwTRv3pwhQ4YAOfcRgDt37jBixAiSkpKwtLRk2rRptGnTBgBXV1eV7ezatQvIKb1X7rNZs2bcuHGDly9fStOU17e5uflb0/YhhYeHf9L9CQXzsfMlPT39jYVRRRphGBsbA6hUH+WmnJ5f+5i3rZ+SkiItU1Sql7Vmist0aht0Y5LL6CIP6gBCovyKLKgDSEyNIyTKr8DLK0sVrKysVB4UCQkJxMXF8ezZM+Li4qQqIVdXV+rXr09qaipz586lWbNmfP3110yaNIkzZ87k2f7169f5+uuv8/w7ffo0gNrqt9yU+839AFZeswkJCbx48YL09HSSkpKIiopi0aJF1K9fn+DgYH799Vcgp3QJYPfu3URGRnLq1ClCQ0OBnCDr9fPh5eWFpaWlSk/y3BYsWEBWVhbjxo17r2vO39+fzMxM2rZti6amJh06dEChULB379533lZucrmcuLg44uLiiI2N5dy5c2zduhXIKa0EePz4MYBUQvI6ZVuqu3fvEh8fL5XKqyt1VQZ1kFNS061bt3z/WVhYvDEvX88HJWV6T58+zYIFC2jbti2XL19m0qRJKBQKaX5AQAD9+vXD3d2dV69eMXXqVCIjI3nyJGdg8ujoaGQyGUuXLqVKlSoEBASwatUqtftUvpTkbvf3PmlXVk9aWlqyaNEimjRpQnBwMBMmTEChUFCrVi1WrlzJy5cvcXd3p2fPnujq6rJu3TrKli0rrZ+enk7jxo1ZtGgRVlZW+Pj44OHhoXafBfE+x6KUlZXFggULkMlkjB8/Ps+xpqam8uOPPzJu3DieP3/O+PHj1ZZob9myhT179qChocGAAQOk6VWrVgVQ265QED4HRVpiZ21tjUwmk258r1M2BK9YsWK+6wNq109MTCQ5OZkaNWp8oNQKRU35Vqw0cuRIqbQG/u1EoKury+bNmwkODubUqVNEREQQGRlJQEAAAQEBjBkzhu+//15ar3Llymo7URTm2sndkzs7+99BqufMmYO1tTU1atSgTZs2HD9+nKysLPr168e5c+cIDg6mc+fOGBoaUq1aNS5fvqwSnAAsX76czMxMhgwZoraUOTg4mJCQEKpXr07z5s3fK/179uwB/i3B6NChAx4eHvj5+TF69Oj3LkWPj4/n66+/VplWsmRJfvrpJzp16qQyXVma9DrludXQ0FBJR+7zrE5GRob0cFfHxMTkjfvLj3K/7u7uNGjQgGbNmnH06FGuXbvGo0ePpPmurq5SgHDt2jX8/f05evSoVOKmra3Nb7/9hq6uLqampgwcOJCgoCCpXWhuSUlJAJQoUeKNaVOmPb/8CggIQC6XY2xsjEwmo2nTpjRo0ICIiAju379PcnIyP/zwA/r6+owYMQJLS0uWLVvG6NGj2bZtm1QiqaGhIQVepqamDB06lP379zNlypQ8+4yPjycjI0P6W9mW+m3edixKu3fv5s6dO7Ru3VplqKsRI0YwcOBA9PT0pJe127dvc/DgQQ4ePCi9MCgUCpYvX46npycaGhrMmDFDpSRYec6VeSAIn5siDez09fVxcHDgxo0bpKWlqZTGZGdnExERgZWVldT49XV169YFcnqkvd4Y9sKFCwB5qmYEaFi1C6HR+0hMjS2S/ZvoW9CgSucCL1+pUiUgJ9BPSkqSbqw//fQTiYmJ3Lx5k8WLF6usI5PJaNy4sdRzLykpicWLF+Pr68umTZtUAjtjY+NCDfVRqlQpQLXk+OXLl9I8ExMTNDQ0kMvl0mDbyg4/mZmZJCUlYWZmxvr167lz5w7Pnj3DwcGB+fPnc/nyZZW2c8r2fMpSNHWU1cWvB0oFFRkZyY0bNwDo06ePyryHDx9y/vx5nJ2d81S95v5bWYr2evtXExMTKa8uXbrEmjVrMDU1VemtWL58eS5fvszff/+tNn3KtFWrVg0zMzNMTU1JSEjg9u3beb5U8/TpU2QyGRYWFly6dCnfnsUA3t7emJubS8ejlDsv1SlZsiRxcXFSBxUDAwNMTU2Ji4vj+fPnUp7n7sCizP/c8w0MDKRzqAx24uPj800vqAY5b0q7ct7rXq8NMTIywsTEhGfPnvHs2TO2bdtGcnIyw4YNk6qn7969i7e3N76+vri7u+cJLq2srICckjV1zWHGjRtHWFiY9Le60rL3ORal/K5/XV3dPC9CyrQ+e/YMyAnq3N3d8fPzw9TUVCrFzO1tgb4gFLUib2PXpUsX5s6dy/bt21WKu/ft20d8fDxjxoyRpkVHR6OjoyOV1Nnb21O9enUCAwMZO3asdLNUKBRs2rQJLS2tPG10BChrWoUu9SYWdTIKzN7enqpVqxIVFcWaNWukXpXKIW1eL8mbN28e58+fZ/LkyTRq1AjIecvu2rUrvr6+ZGRkfNCbs4ODA1paWlIDavg3+KhTpw56eno4ODhw/fp1Ll26ROPGjblz5w6Q8yA1MzMjKiqK4OBgKlasSPPmzcnKyiIkJASZTKbSYF/Z27JBgwaYmZnlSUtycjInTpwAkBr/vytlaV316tWpXv3f3tvR0dFcunQJPz8/nJ2dMTc3x9zcnOfPnxMaGqrSPkmZhpo1a6psW1tbW3pQNm7cmLCwMC5cuMCSJUuYOnWqNP3gwYMcOXKEK1euqAxndPbsWS5cuICenh6urq7IZDI6duzI5s2bWbt2LU5OTlJpjFwuZ9q0afz111/MmTOHFi1asG7dunyP287OTgq+8stLgEePHpGamkrp0qUpUaIEderU4fDhw0RERGBvb8+LFy948eIFMpmMChUqSOspG+8DUv5XrFgRGxsbzMzMiI+P586dO1SuXFmar67tJ/xbapS7BPL1hvwKhUIKmpRpULbZLFu2LOnp6UyaNImUlBS8vLzQ1dXl6dOnxMfHI5PJqFy5svTbyl3CphzDLTs7m7CwMDw8PKhQoQJz584Fcl4MAGxsbNSWrk2ZMkWqas3Pux6LssfzvXv3uHbtGvr6+jRt2lRlm8uWLSMkJIThw4dLQ+so81ZZjb9s2TL8/PyoWLEi69evVzsagzLAfFtpqSAUlSIP7Hr27ElAQAALFy4kJiYGR0dHbt++zcaNG7G3t2fQoEHSsq6urlSqVInAwEBp2owZM+jXrx99+vShf//+lChRgoCAAMLCwhg3bly+N0bhf4dMJmPBggUMHDiQjRs3Eh0dTZMmTZDL5Vy5coUjR44A/7a9srKywsvLi3HjxtG1a1esra1JSkqS2od98803Kg+c+Ph4du7cqXbfnTt3JioqiqCgIODfDgO7du3i7NmzUpVqx44d8fPz48cff8TKyooDBw5QqlQp2rdvD8DQoUMZP34806dPp0+fPgQEBAA5A2lDTkC2YMECTExMGDp0KOfOnePRo0d8++23KiXWly5dAtSP3Qg54/dlZGRQunTpfEu63yQzMxN/f38gp5do7gbAd+/epW3btgQGBvLLL79gYGDA1KlTmThxIlOnTuXq1auULVuW8+fPc/jwYUxNTRk1alS++5LJZMyePZvOnTuzefNmqUdwhQoVmDBhAkuXLqVv37707t0ba2tr7t69y7Zt29DS0mLOnDnSGHHjxo3j3LlzhIaG0rNnTzp06EB2djZBQUFcvXqV6tWr07p16wKVzJqamuLs7ExISAjz588nOzub8PBwbG1t+eqrrwD48ccfCQsLY+bMmfTq1YtBgwZx7NgxlixZQlJSEmfOnCErK4tOnTphampKkyZNsLOzIyIiQhqX8Pjx45QoUYJOnTqhoaHB4MGDWbRoEePGjZN6IcO/18frlM1TlAEg5ATENjY27Nu3jzJlynD//n0ePXpE48aNpcBlwIABxMTEsG7dOpo0aUJ2djaXLl1i5MiRNGnShP379yOXy+nevTtmZma4uroSGBjI5s2biYuL4+zZs+zfvx8NDQ3atm1L5cqViY6OJiwsDC0tLaysrNi4cSOA2iGEIP9rN7f3ORb49/dhZ2eXp3SuUqVKeHp6Mn36dO7evcv9+/cJDg6mVKlSdO3alZs3b/Lnn38C0KRJE0JDQ6V2roaGhtKLS3R0NIDanrmC8Dko8sBOR0eHDRs2sHr1agIDA9m2bRvm5ub07NmTsWPHvnXsqS+++IJt27axcuVKPDw8yMzMpEqVKvz222+itK4YcXR0xM/Pj3Xr1vHXX38RGhqKpqYmFhYWuLq60qVLF2nA4P79+2NhYYGvry/+/v4kJCSgo6NDpUqVcHd3z/OwfPDgQb7fFHZxcSEyMhJPT0+V6covmnTp0oU2bdpIY3idOXOG5ORknJ2dcXd3lzrvtGvXThrXbPny5ZiamjJ27FhGjBgB5FzHv/32G56entL8wYMHqzT+hpyenZB/taByCJL85r/NqVOniI+Pp1KlSnl6dVWqVIk6depw6dIlaUy79u3bY2RkxIYNG9i/fz/JycmULVuWvn37MnTo0HwHk1WqXLkyo0ePZtmyZfz000/SeR0+fDg1atTA29ubPXv2kJycTMmSJWndujWDBw9WKQk0NjZm+/btbNq0iaCgIFauXElWVhbly5dn3Lhx9OvX750GKF6+fDmzZ88mICCA7OxsWrZsybRp0/K0dVSqU6cOnp6erFixAg8PDwwMDOjXrx+TJk0CQEtLi/Xr17NgwQKOHj1KWloa9erVY/r06VKp65AhQ9DU1GTLli0sW7YMCwsLZs6cqXZ8QcgJjrS0tLh27RoZGRno6Oigra3NH3/8wdy5c/Hx8UFbW5tOnTq98XvZK1asYOXKlZw8eZKwsDBKly7N+PHjpZ6kLi4uLF68GC8vL/z9/cnOzsbe3p6RI0dKzVy8vb1ZunQpR44cISkpicqVKzNjxgzatm1b4HP+uvc5Fnjz9e/m5oZCocDLy4vVq1ejqamJi4sLU6ZMwcTEhGPHjkntIZUDkytZWVlJgZ1yGIvCjNMnCB+TTPEfbzCg7Db8sYY7URLd0j9P/+V8UX45Qvl1iM/FfzlP3sWYMWM4fPgwCxcupHPngrdZfV8iX3La+bZq1Qpzc3NOnDghDcis/LJKly5dWLBgwSdLj8iTz9OnGu4kv7il6MfeEAShSL169YqdO3dK1cPC/4b+/fsDOV+gyK8HsfBhbdy4EblcTp8+fdDW1ubJkyfs3LlTGndPED4HIrAThP+4hIQEpk2bxsKFC4s6KcI7cHJywtXVlb///lvt916FD+v27dts27YNKysrqe33rVu3mDZtmvicmPBZKfI2doIgFI3y5curHWpC+N+xbNkyli1bVtTJ+E+oVq0a169fV5nWpEkT8RsSPjuixE4QBEEQBKGYEIGdIAiCIAhCMSECO0EQBEEQhGJCBHaCIAiCIAjFhAjsBEEQBEEQigkR2AmCIAiCIBQTIrATBEEQBEEoJkRgJwiCIAiCUEyIwE4QBEEQBKGYEIGdIAiCIAhCMSECO0EQBEEQhGJCBHaCIAiCIAjFhFZRJ6CoKRQKADIyMj76vtLT0z/6PoR3J/Ll8yPy5PMk8uXzI/Lk8/Qx80UZryjjl9fJFPnN+Y94+fIlt27dKupkCIIgCIIgFJitrS3GxsZ5pv/nAzu5XM6rV6/Q1tZGJpMVdXIEQRAEQRDypVAoyMzMxNDQEA2NvC3q/vOBnSAIgiAIQnEhOk8IgiAIgiAUEyKwEwRBEARBKCZEYCcIgiAIglBMiMBOEARBEAShmBCBnSAIgiAIQjHxnx+guDCSkpLw8PDg2LFjxMbGYmpqStOmTRk/fjylS5d+6/oRERGsXr2aiIgI0tPTqVixIj169KB3795quzALBVPYfLlw4QJr167lxo0bvHr1Cmtra9q2bcugQYPQ09P7BEdQ/BQ2T3JLT0+nU6dO3Lt3D29vb7788suPlOrir7D5kpGRgaenJ/7+/jx58gRzc3OaNm3KuHHjMDMz+wRHUPwUNk+OHTvGpk2biI6OJjU1lfLly9OyZUsGDhyIiYnJJziC4ikjI4Ply5ezYcMG6tevz+bNmwu87qd+1ovhTt5TSkoKPXv2JDo6mj59+lCzZk3u3bvHhg0bMDc3Z9euXZQsWTLf9UNCQhg6dChlypThu+++w9TUlMOHD3Ps2DH69u3LtGnTPuHRFB+FzZeDBw8yceJEbGxs6NmzJ0ZGRpw+fZqgoCDq1KnD1q1bRdD9jgqbJ69btmwZnp6eACKwK4TC5ktWVhZDhgzhwoUL9O3bF3t7e/7++282b95M5cqV2bNnDzo6Op/wiP73FTZPlL8NR0dHunTpgr6+PhEREezevZty5crh5+eHkZHRJzyi4uHOnTtMnjyZu3fvkpKSgrOzc4EDuyJ51iuE9+Lp6amwtbVVbNmyRWX64cOHFba2tor58+fnu65cLle4uLgo6tWrp3j69KnKvBEjRijs7OwUN27c+CjpLu4Kky/p6emKunXrKpo1a6ZISkpSmff9998rbG1tFSdOnPgYyS7WCpMnr4uMjFTUqFFD0blzZ4Wtra0iNDT0Qyf3P6Ow+eLj46OwtbVVHDx4UGX6qlWrFC1atFCEhYV98DQXd4XJk/j4eEX16tUVzZs3V6SlpanMW7JkicLW1laxcePGj5HsYi0hIUFRu3ZtRadOnRTR0dEKW1tbxXfffVegdYvqWS+KHt6Tv78/BgYGdO3aVWV6q1atKFOmDP7+/vl+x+3atWvcvXuXdu3aYWFhoTKvb9++KBQK9u/f/9HSXpwVJl+ePXtG69atGTZsWJ7PtDRu3BhAfH7uPRQmT3KTy+X88ssvWFlZ0bNnz4+V3P+MwubLli1bcHBwoF27dirTR48ezbFjx6hfv/5HSXdxVpg8efLkCVlZWTg6OqKrq6syr169egA8evTo4yS8GMvMzKRTp07s2LGDypUrv9O6RfWsF4Hde3j16hW3b9/GwcEhT1WDTCajdu3aPHv2jIcPH6pd//LlywDUqlUrz7zatWurLCMUXGHzpVy5cixYsIBevXrlmffy5UsAtd/lE/JX2DzJzcfHhytXrjB79mxRxVdIhc2Xp0+fEh0dzddffy1NS09PRy6Xf9R0F2eFzRNra2t0dHS4d+9ennnKdapUqfLB013clSpVil9//TVPsFwQRfWsF4Hde3jw4AEAZcuWVTu/TJkyKsu9y/qGhoaUKFEi33WF/BU2X/KTmprKzp070dHRoUWLFoVL5H/Mh8qTx48fs2zZMrp164azs/OHTeR/UGHzJTo6GoAKFSqwfv16mjVrRq1atahVqxbDhw/n7t27HyHVxVth88TIyIgRI0YQGRnJrFmziI6O5unTpxw+fJg1a9Zga2tL586dP07iBbWK6lkvesW+h1evXgGgr6+vdr5yenJy8nuvn9+6Qv4Kmy/qyOVypk2bxt27d5k4cSKWlpaFT+h/yIfKk5kzZ2JoaMiUKVM+bAL/owqbLwkJCUBOdSzA2LFjMTExITQ0lC1btnD58mX27dsnfi/v4EP8VkaPHo2ZmRnz5s1j69at0vTmzZuzYMEC0av/EyuqZ70I7D4imUz2XuspFIr3Xld4u4Ke27S0NCZNmsTRo0fp1q0bw4YN+8gp++96U54cOHCAkydPsmLFCkqUKPEJUyXkly+ZmZlAThOFgIAADAwMAGjZsiWlS5dmyZIlbNiwAXd390+W1v+KN/1WfHx8mDdvHk2aNKFjx47o6+tz+fJlvL29GTZsGOvWrRNDnnxGPtazXlTFvgdlOytlNP465fT8upW/bf2UlBTRlus9FDZfcouPj6d///4cPXqUESNGMHv2bBFsv4fC5kliYiLz5s2jefPmtG3b9uMk8j+osPmiDOSaNWsm/V+pS5cuAJw/f/6DpPW/orB5Eh0dzbx582jUqBGenp60b9+eFi1aMGHCBObPn8/ly5f5/fffP07iBbWK6lkvSuzeg7W1NTKZjMePH6udHxMTA0DFihXzXR9Qu35iYiLJycnUqFHjA6X2v6Ow+aL0/PlzevfuTUxMDL/99htubm4fOqn/GYXNk99++42UlBRGjRrFkydPpOlJSUlATgD+5MkTzMzMRIeKd1DYfClfvjyA2jEdzc3Nkclk+T7MBPUKmychISFkZ2fTsmXLPPOaN2+OTCYjLCzswyVYeKuietaLwO496Ovr4+DgwI0bN0hLS1Npt5CdnU1ERARWVlaUK1dO7fp169YFckaj7tGjh8q8CxcuAODk5PSRUl98FTZfIKf9yuDBg3ny5Al//PEHX3311adIerFV2DwJDQ0lJSWFbt26qZ0/fvx4QAxU/K4Kmy9Vq1bF2NiYmzdv5pn36NEjFApFnuEdhDcrbJ6kpaUBOb2TX5eeno5CoZCq0IVPo6ie9aIq9j116dKFtLQ0tm/frjJ93759xMfH880330jToqOjVXq+2NvbU716dQIDA1UieYVCwaZNm9DS0hKlRO+pMPkCMHfuXCIjI1m6dKkI6j6QwuTJ3Llz8fT0zPOvf//+AEycOBFPT09sbW0/zcEUI4XJF21tbTp16kRYWJj0gFLy8fEBoGnTph8x9cVTYfLkiy++AODQoUN5xro7cuSIyjLCx/G5POvFJ8XeU0ZGBt999x3Xrl2jT58+ODo6cvv2bTZu3EiVKlXYtm2b1PbEzs6OSpUqERgYKK0fERFBv379KFWqFP3796dEiRIEBAQQHBzMuHHjGDVqVFEd2v+0wuRLZGQkbm5u2Nra5nv+zczMxHAb76iwvxV19uzZg7u7uyipK4TC5kt8fDzdu3cnISGBQYMGUaZMGc6ePYu/vz92dnb4+vrm2xtQUK+weTJmzBgOHz5MnTp1aN++PUZGRly/fh1fX1+MjY3ZtWvXG2sshLyioqKIioqS/h43bhxVq1ZlzJgx0rSmTZuir6//2TzrRWBXCMnJyaxevZrAwEDi4uIwNzendevWjB07VqX3Xn4Pq+vXr7Ny5UrCw8PJzMykSpUq9OvXT5TWFdL75osyWHiTd/lGoPCvwv5WXicCuw+jsPny/PlzVqxYwfHjx0lISKB06dK0atWKMWPGiB7M76kweZKVlcXWrVvZu3cvd+7cISsrCwsLC77++mtGjRoljYUnFJyHhwerVq164zLHjh2jfPnyn82zXgR2giAIgiAIxYRoYycIgiAIglBMiMBOEARBEAShmBCBnSAIgiAIQjEhAjtBEARBEIRiQgR2giAIgiAIxYQI7ARBEARBEIoJEdgJgiAIgiAUEyKwEwThozt37hx2dnZ4eHgUdVKKlIeHB3Z2dpw7d+6DLCcIgvA6raJOgCAIn4eCfHkDYP78+SrfrPxf0LdvX8LCwvJM19XVpWzZsjRr1ozhw4djZmb2UdPRrl07qlWrRrVq1aRpt27d4uzZswwYMOCNy30qDx8+pGXLlmrnaWpqYmZmxhdffMHgwYOpU6dOofa1Zs0aOnXqRPny5Qu1HUEQ/iUCO0EQVHTs2JFWrVrlO9/R0fETpubDmjNnDsbGxtLfz58/Jzw8HG9vb4KCgtixYwcWFhYfbf9Vq1alatWqKtMCAwPZu3evSmCnbrlPzcHBgREjRqhMS0tL4/bt2/j6+nLs2DGWLFmCq6vre23/wYMHrFixgnr16onAThA+IBHYCYKgwtbWlrZt2xZ1Mj6KZs2aUbp0aZVpffr0oVatWsyfPx9PT0+mT5/+SdN09erVT7q/gjI3N8/3Ovj222/55ptvmDNnDi4uLmhqar7z9j/X4xaE/3WijZ0gCO/t7t27uLu706JFC2rVqsXXX3/NsGHDuHLlylvXTU5OZtWqVbRv3546depQr1493Nzc2LRpE3K5XGXZ+Ph4Zs+eTfPmzalZsyZffvklI0aMICIi4oMcR5cuXQC4cOGCyvTo6GgmTpxIo0aNqFGjBl999RVjx44lMjJSZbnMzEw2bdqEm5sbTk5O1KlTB1dXV1auXElGRoa0XO62cw8fPsTOzo7Tp08TExODnZ0dffv2zbPc/fv3sbOzY/DgwWrT/vvvv2NnZ8fu3bulaefPn2fw4MHUr1+fmjVr0rp1axYsWEBSUtIHOV+VK1emXr16PH/+nLt376rMO3jwIH379qVhw4bUqlULFxcXfvvtN5V99+3blwkTJgDQr18/7OzsePjwoTT/yJEj9O7dm7p161KrVi1cXV1ZtWqVyrkUBEE9UWInCMJ7iY2NpXfv3mRmZjJw4EAqVKhAbGwsXl5e9O7dm23btr2x2nb8+PEEBwfTs2dPvvjiC7KzswkODmb+/Pk8fvxYau+XkJBA9+7defHiBX369KFy5co8ffqU7du3891337Fu3ToaNmxYqGPR0dFBJpORlZUlTbt16xa9evVCQ0OD/v37Y21tzZ07d/D29ub06dNs3bqV6tWrAzlVvNu3b6d9+/b07dsXTU1NLl68yO+//86tW7dYtWpVnn2am5uzYsUKZs2aBcCMGTPUtvGrWLEijo6OnDt3jsTERExMTFTmBwYGoquri4uLCwCHDx9m/Pjx2NnZMWbMGIyMjIiIiMDHx4e//vqLHTt2oK+vX6jzBaCtrQ2gUlq3c+dOpk2bRq1atRg7diz6+vpcvnyZjRs3EhYWxs6dO9HQ0GDMmDFs2bKFwMBAxowZQ9WqVTE3NwfAy8uLefPm8eWXXzJp0iR0dHQIDQ1l1apVhIeHs379ejQ0RJmEIORHBHaCILyX6OhoatasSYcOHejcubM0vVq1agwdOpTt27fnG9glJCRw5swZmjZtysyZM6Xp3377LWXKlOHFixcoFApkMhmrV68mJiYGX19fatWqJS3r5uZG+/btWbBgAfv27SvUsZw9exaFQkGNGjWkaQsXLiQ5OZnt27erdBJo3Lgxffr0YcmSJaxfvx6AgIAAqlWrxtKlS1XSV7FiRS5dukRKSgoGBgYq+9TX16dt27YsXLgQ4I3V3x06dODq1ascP35cKl0EuHfvHpGRkbRr1w4jIyMyMjKYOXMmdnZ27NixQwq+vvnmG2xtbZk9eza+vr4q7fneR0JCApcuXcLExARra2tp+sOHD2nQoAHLly+nZMmS0nlISEjg4MGDXLx4EScnJ5ydnaUev/Xr1+fLL78E4NmzZyxatIjmzZvj6ekpbbdbt25YWFiwYcMGjh8//sY2oILwXycCO0EQVKSnp7+xys7Q0BBNTU0aNmwolZTJ5XJSUlKQy+VYWVkBEBMTk+82NDU10dDQIDo6mvj4eJWSqh9//FFl2UOHDlGhQgVsbGxU0qWvr4+TkxMnTpwgNja2QJ0ekpOT0dXVlf5OSEjg/PnzLFmyBAMDA4YPHw5ASkoKf/31F/b29nl6fjo5OVG1alVCQ0NJT09HV1cXLS0tHj9+zIMHD1QCnfyqT9+Vq6srv/32G0FBQSqB3aFDhwDo1KkTAGFhYTx//pxevXqRmppKamqqtGyLFi2YO3cuZ86cKVBgl52dnec6SEtL49atWyxfvpyEhAR++eUXtLT+fYwoq1cBMjIySE9PR6FQULFiRSDnmnBycsp3n8eOHSMzM5NWrVrl2Xfr1q3ZsGEDp0+fFoGdILyBCOwEQVCxatUqtVWHSnv37sXBwQHIGSLFx8eHmzdvqlRjQk5gkB9jY2N69erFli1baNOmDU2bNqVhw4Y0btwYS0tLabnExETi4uKIi4ujfv36+W7v8ePHBQrs8isVc3R0ZPr06VJP1Pv37yOXy/PtmVqpUiWioqJ4+PAhVapUYdiwYSxcuJAOHTrQuHFj6VgqVKjw1jQVhIWFBfXr1+evv/4iOTkZIyMjIKca1tTUlMaNGwMQFRUFvDkPHz16VKB9hoSE5HvObWxsmDdvHt9++63K9MTERFauXMnhw4eJjY3Ns96brgmA27dvAzB16lSmTp2qdpmCpl8Q/qtEYCcIgoru3bvToUOHfOcrg5WtW7cya9YsrK2t+fnnn7G2tkZXV5fExETGjBnz1v1Mnz4dJycnduzYQWBgIAEBAchkMr7++mtmzJiBtbW1VOJkZ2eX74MechrzF4SHh4dKGzUdHR3KlSunEkwCvHr1CiBP9amSso1aSkoKkFMyZ29vz+bNmzl9+jRHjhwBoE6dOvzyyy8qVbzvq2PHjpw7d46TJ0/SoUMHqRq2d+/eUpWrMj1DhgyhSZMmarejp6dXoP05OjoyZcoUlWmLFi3i77//ZsWKFdjb26vMUygUDBs2jIiICFxcXHBxccHMzAwNDQ0CAgLYsWPHW/epzG93d3fp5eF1r7cxFARBlQjsBEFQYW1tLbV5ehMvLy80NTXZtGmTyjhkDx48KPC+XF1dcXV1JTk5mXPnzrFv3z6CgoIYPnw4+/fvx9DQEMjpdVqQNL1NnTp18gx3oo5yv8pA6XXKAES5HECjRo1o1KgRaWlphIeHc+DAAfz8/BgyZAiBgYGFDkjatGnDrFmzOHz4MB06dMhTDZs7PSYmJoU+X+q2MXv2bL799lt+/vlnduzYoVINe+nSJSIiImjYsCErV65UWS80NLRA+1Sm38LC4oPktyD8F4muRYIgvJeYmBjKlCmTZ3BZdV94eBsjIyNatmzJypUrad++PdHR0URFRWFsbIylpSUPHjwgPj4+z3rqpn0INjY2aGlpSVWDr7t9+zY6OjpqB9bV09OjUaNGzJs3jyFDhhAfH59nGJX3YWJiQuPGjTl9+jQZGRkEBQVRoUIFlTaAyi9V5DfcTGHPl4ODA/369eP69ets2LBBZZ6yilRdG7rz588XaPvK9F++fDnPvIyMDJKTk981yYLwnyMCO0EQ3ouFhQUvXrxQaaD/7NkzNm3ahEwmIy0tLd91T5w4QYsWLQgODs4zT0dHB/h3OI127dqRmZnJli1bVJZLTEzEzc1N6vDwIenr69OkSRNu3rzJxYsXVeadO3eOe/fu0bx5c3R0dLh69SouLi5qqxqVx5K7ZOt1GhoapKenFyhdHTp0IDU1lb1793Ljxg06duyoMt/JyQlzc3NOnDjBvXv3VOYdPHiQr7/+moMHDxZoX/kZO3Ys5cqVY9WqVURHR0vTlSWhr3ea8ff359atWwAq14RyyJLcx96iRQt0dHTYu3cvL168UNmOl5cXDRs2JDw8vFDpF4TiTlTFCoLwXlxdXVm3bh1jxoyhQ4cOPH78mG3btjFp0iSWLFnCjRs32Lp1K82aNcuz7hdffEFmZiZjx46lV69eVKtWDblcTkREBHv37qVBgwZUqVIFgJEjR3Ls2DHWrFlDXFwcTk5OPHv2jO3btxMfH8933333UY7vhx9+IDw8nJEjRzJgwACVcexKliwptT+zt7dHR0eHX3/9lcjISGrWrImGhgY3b95ky5YtVKlS5Y3j7JUvX56QkBDmz59PmTJlGDhwYL7LtmjRAgMDA1asWAGoVsNCTiA5c+ZMxo8fz3fffceAAQMoVaoU165dw9fXl0qVKqnNj3dhYGDA9OnTGTFiBFOnTmXr1q1oaGhQp04drKys8Pf3x9LSEhsbG86fP09wcDC//PILkydPxs/PD1NTU1xdXaXSTk9PT6KiomjatClVqlRh8uTJzJs3j549e/Ldd99hZGTE+fPn8fPzo379+oX+Pq0gFHcisBME4b18//33ZGRkEBgYyIwZM6hatSpTp07FxcWFjIwMFi1axPLly9V+yL5kyZJs376dtWvXEhgYiI+PD9ra2lhZWTFp0iSVYM3U1JQdO3awevVqTpw4wZ49eyhRogS1atViwYIF1K1b96McX6VKldixYwceHh54e3uTlJREyZIladWqFaNHj5aGNdHW1sbb25u1a9dy4sQJ/Pz8AChXrhwDBw5kwIABUsmdOuPGjSMmJoYdO3bg4ODwxsBOX1+fFi1aEBAQQK1atbCxscmzTJs2bfDy8mLt2rWsXbuWtLQ0LC0t6dOnDyNGjMi3Q8i7aN68OS4uLgQFBeHt7S0d49q1a5kzZw7e3t5oa2vToEEDtm7diqWlJQEBAYSEhPDHH3/g6upK27ZtOXToEKGhoTx58oTatWsD0L9/f6ysrNi0aRPLli0jOzsbKysrRo8ezdChQ8XgxILwFjKFQqEo6kQIgiAIgiAIhSdefQRBEARBEIoJEdgJgiAIgiAUEyKwEwRBEARBKCZEYCcIgiAIglBMiMBOEARBEAShmBCBnSAIgiAIQjEhAjtBEARBEIRiQgR2giAIgiAIxYQI7ARBEARBEIoJEdgJgiAIgiAUE/8HZ7HenuyxdQsAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 10)) \n",
    "draw_roc(val_labs_20271_20194_31519, probs[0], index = 0) \n",
    "draw_roc(val_labs_106977, probs[1], index = 1) \n",
    "\n",
    "plt.plot([0,1], [0,1], color='orange', linestyle='--', linewidth = 3)\n",
    "\n",
    "plt.xticks(np.arange(0.0, 1.1, step=0.2), fontsize = 20)\n",
    "plt.xlabel(\"False Positive Rate\", fontsize = 20)\n",
    "\n",
    "plt.yticks(np.arange(0.0, 1.1, step=0.2), fontsize = 20)\n",
    "plt.ylabel(\"True Positive Rate\", fontsize = 20)\n",
    "\n",
    "    #plt.title('ROC Curve', fontweight='bold', fontsize=15)\n",
    "plt.legend(prop={'size':15, 'weight':'bold'}, loc='lower right')\n",
    "plt.savefig('/home/shayantan/TNBC-single-cell/Reproducibility_best_results/ROC_plot_external_validation.pdf')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 10)) \n",
    "draw_prc(val_labs_20271_20194_31519, probs[0], index = 0) \n",
    "draw_prc(val_labs_106977, probs[1], index = 1) \n",
    "\n",
    "plt.xticks(np.arange(0.0, 1, step=0.3), fontsize = 20)\n",
    "plt.xlabel(\"Recall\", fontsize = 20)\n",
    "\n",
    "plt.yticks(np.arange(0.0, 1, step=0.3), fontsize = 20)\n",
    "plt.ylabel(\"Precision\", fontsize = 20)\n",
    "\n",
    "    #plt.title('ROC Curve', fontweight='bold', fontsize=15)\n",
    "plt.legend(prop={'size':15, 'weight':'bold'}, loc='lower right')\n",
    "plt.savefig('/home/shayantan/TNBC-single-cell/Reproducibility_best_results/PRC_plot_external_validation.pdf')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_roc(y_test, y_probs, index):\n",
    "    names = ['GSE20271-GSE20194-GSE31519','GSE106977']\n",
    "\n",
    "    fp_rate, tp_rate, thresh = roc_curve(y_test, y_probs)\n",
    "    auc = roc_auc_score(y_test, y_probs)\n",
    "    n_bootstraps = 1000\n",
    "    rng_seed = 42  # control reproducibility\n",
    "    bootstrapped_scores = []\n",
    "    rng = np.random.RandomState(rng_seed)\n",
    "\n",
    "    for k in range(n_bootstraps):\n",
    "        # bootstrap by sampling with replacement on the prediction indices\n",
    "        indices = rng.randint(0, len(y_probs), len(y_probs))\n",
    "        if len(np.unique(y_test[indices])) < 2:\n",
    "            continue\n",
    "        score = roc_auc_score(y_test[indices], y_probs[indices])\n",
    "        bootstrapped_scores.append(score)\n",
    "                #print(\"Bootstrap #{} ROC area: {:0.3f}\".format(k + 1, score))\n",
    "    sorted_scores = np.array(bootstrapped_scores)\n",
    "    sorted_scores.sort()\n",
    "    confidence_lower = sorted_scores[int(0.05 * len(sorted_scores))]\n",
    "    confidence_upper = sorted_scores[int(0.95 * len(sorted_scores))]\n",
    "    plt.plot(fp_rate, tp_rate, label=names[index]+\" [AUROC=\" +str(round(auc,3)) + \n",
    "                     \" (\"+ str(round(confidence_lower,3))+\" - \"+str(round(confidence_upper,3))+\")]\", linewidth = 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import average_precision_score, precision_recall_curve\n",
    "\n",
    "\n",
    "def draw_prc(y_test, y_probs, index):\n",
    "    names = ['GSE20271-GSE20194-GSE31519','GSE106977']\n",
    "\n",
    "    precision, recall, thresh = precision_recall_curve(y_test, y_probs)\n",
    "    auprc = metrics.auc(recall, precision)\n",
    "    n_bootstraps = 1000\n",
    "    rng_seed = 42  # control reproducibility\n",
    "    bootstrapped_scores = []\n",
    "    rng = np.random.RandomState(rng_seed)\n",
    "\n",
    "    for k in range(n_bootstraps):\n",
    "        # bootstrap by sampling with replacement on the prediction indices\n",
    "        indices = rng.randint(0, len(y_probs), len(y_probs))\n",
    "        if len(np.unique(y_test[indices])) < 2:\n",
    "            continue\n",
    "        score = average_precision_score(y_test[indices], y_probs[indices])\n",
    "        bootstrapped_scores.append(score)\n",
    "                #print(\"Bootstrap #{} ROC area: {:0.3f}\".format(k + 1, score))\n",
    "    sorted_scores = np.array(bootstrapped_scores)\n",
    "    sorted_scores.sort()\n",
    "    confidence_lower = sorted_scores[int(0.05 * len(sorted_scores))]\n",
    "    confidence_upper = sorted_scores[int(0.95 * len(sorted_scores))]\n",
    "    plt.plot(recall, precision, label=names[index]+\" [AUPRC: \" +str(round(auprc,3)) + \" (\"+ str(round(confidence_lower,3))+\" - \"+str(round(confidence_upper,3))+\")]\", linewidth = 4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Integrated model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 492,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['S100B', 'NOL7', 'ILF2', 'HAT1', 'FBXO16', 'LYPLA1', 'TMEM14B', 'PDCL3',\n",
      "       'S100P', 'DKK3', 'PTPRC', 'GREM1', 'PTH2R', 'CSN3', 'SAA1', 'NOP10',\n",
      "       'TUBB6', 'B-cells', 'CD4+ T-cells', 'CD4+ Tcm', 'CD4+ Tem',\n",
      "       'CD4+ memory T-cells', 'CD4+ naive T-cells', 'CD8+ T-cells', 'CD8+ Tcm',\n",
      "       'CD8+ Tem', 'CD8+ naive T-cells', 'Memory B-cells', 'naive B-cells',\n",
      "       'pro B-cells', 'Tregs', 'ImmuneScore', 'StromaScore',\n",
      "       'MicroenvironmentScore'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "#TNBC_subtype + purity + enrichment scores - training data\n",
    "val_data_feats_25066_subtype_xcell_estimate = pd.read_table(\"/home/shayantan/TNBC-single-cell/Reproducibility_best_results/TNBC_subtype_Cell_enrichment/GSE25066_processed_tnbc_type_xcell_estimate.txt\")\n",
    "cols = top_feats_after_rfecv[0:17] + [\"B-cells\",\"CD4+ T-cells\",\"CD4+ Tcm\", \"CD4+ Tem\",\"CD4+ memory T-cells\", \"CD4+ naive T-cells\",\"CD8+ T-cells\",\"CD8+ Tcm\",\"CD8+ Tem\",\"CD8+ naive T-cells\",\"Memory B-cells\",\"naive B-cells\", \"pro B-cells\",\"Tregs\", \"ImmuneScore\",\"StromaScore\", \"MicroenvironmentScore\"]\n",
    "numeric_cols = top_feats_after_rfecv[0:17] + [\"B-cells\",\"CD4+ T-cells\",\"CD4+ Tcm\", \"CD4+ Tem\",\"CD4+ memory T-cells\", \"CD4+ naive T-cells\",\"CD8+ T-cells\",\"CD8+ Tcm\",\"CD8+ Tem\",\"CD8+ naive T-cells\",\"Memory B-cells\",\"naive B-cells\", \"pro B-cells\",\"Tregs\", \"ImmuneScore\",\"StromaScore\", \"MicroenvironmentScore\"]\n",
    "val_data_feats_25066_subtype_xcell_estimate = val_data_feats_25066_subtype_xcell_estimate[cols]\n",
    "#le = LabelEncoder()\n",
    "#le.fit(val_data_feats_25066_subtype_xcell_estimate['TNBC_type'])\n",
    "#val_data_feats_25066_subtype_xcell_estimate['TNBC_type'] = le.transform(val_data_feats_25066_subtype_xcell_estimate['TNBC_type'])\n",
    "#val_data_feats_25066_subtype_xcell_estimate_encoded = pd.get_dummies(val_data_feats_25066_subtype_xcell_estimate, columns=['TNBC_type'])\n",
    "#val_data_feats_25066_subtype_xcell_estimate_encoded = (val_data_feats_25066_subtype_xcell_estimate, columns=['TNBC_type'])\n",
    "sc = MinMaxScaler(); val_data_feats_25066_subtype_xcell_estimate_encoded = val_data_feats_25066_subtype_xcell_estimate\n",
    "val_data_feats_25066_subtype_xcell_estimate_encoded[numeric_cols] = sc.fit_transform(val_data_feats_25066_subtype_xcell_estimate_encoded[numeric_cols])\n",
    "val_data_feats_25066_subtype_xcell_estimate_encoded = pd.DataFrame(val_data_feats_25066_subtype_xcell_estimate_encoded, columns = val_data_feats_25066_subtype_xcell_estimate_encoded.columns)\n",
    "print(val_data_feats_25066_subtype_xcell_estimate_encoded.columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['S100B', 'NOL7', 'ILF2', 'HAT1', 'FBXO16', 'LYPLA1', 'TMEM14B', 'PDCL3',\n",
      "       'S100P', 'DKK3', 'PTPRC', 'GREM1', 'PTH2R', 'CSN3', 'SAA1', 'NOP10',\n",
      "       'TUBB6', 'B-cells', 'CD4+ T-cells', 'CD4+ Tcm', 'CD4+ Tem',\n",
      "       'CD4+ memory T-cells', 'CD4+ naive T-cells', 'CD8+ T-cells', 'CD8+ Tcm',\n",
      "       'CD8+ Tem', 'CD8+ naive T-cells', 'Memory B-cells', 'naive B-cells',\n",
      "       'pro B-cells', 'Tregs', 'ImmuneScore', 'StromaScore',\n",
      "       'MicroenvironmentScore'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "#TNBC_subtype + purity + enrichment scores - training data\n",
    "val_data_feats_combined_subtype_xcell_estimate = pd.read_table(\"/home/shayantan/TNBC-single-cell/Reproducibility_best_results/TNBC_subtype_Cell_enrichment/GSE20271_20194_31519_processed_tnbc_type_xcell_estimate.txt\")\n",
    "cols = top_feats_after_rfecv[0:17] + [\"B-cells\",\"CD4+ T-cells\",\"CD4+ Tcm\", \"CD4+ Tem\",\"CD4+ memory T-cells\", \"CD4+ naive T-cells\",\"CD8+ T-cells\",\"CD8+ Tcm\",\"CD8+ Tem\",\"CD8+ naive T-cells\",\"Memory B-cells\",\"naive B-cells\", \"pro B-cells\",\"Tregs\", \"ImmuneScore\",\"StromaScore\", \"MicroenvironmentScore\"]\n",
    "numeric_cols = top_feats_after_rfecv[0:17] + [\"B-cells\",\"CD4+ T-cells\",\"CD4+ Tcm\", \"CD4+ Tem\",\"CD4+ memory T-cells\", \"CD4+ naive T-cells\",\"CD8+ T-cells\",\"CD8+ Tcm\",\"CD8+ Tem\",\"CD8+ naive T-cells\",\"Memory B-cells\",\"naive B-cells\", \"pro B-cells\",\"Tregs\", \"ImmuneScore\",\"StromaScore\", \"MicroenvironmentScore\"]\n",
    "val_data_feats_combined_subtype_xcell_estimate = val_data_feats_combined_subtype_xcell_estimate[cols]\n",
    "#le = LabelEncoder()\n",
    "#le.fit(val_data_feats_25066_subtype_xcell_estimate['TNBC_type'])\n",
    "#val_data_feats_25066_subtype_xcell_estimate['TNBC_type'] = le.transform(val_data_feats_25066_subtype_xcell_estimate['TNBC_type'])\n",
    "#val_data_feats_25066_subtype_xcell_estimate_encoded = pd.get_dummies(val_data_feats_25066_subtype_xcell_estimate, columns=['TNBC_type'])\n",
    "#val_data_feats_25066_subtype_xcell_estimate_encoded = (val_data_feats_25066_subtype_xcell_estimate, columns=['TNBC_type'])\n",
    "sc = MinMaxScaler(); val_data_feats_combined_subtype_xcell_estimate_encoded = val_data_feats_combined_subtype_xcell_estimate\n",
    "val_data_feats_combined_subtype_xcell_estimate_encoded[numeric_cols] = sc.fit_transform(val_data_feats_combined_subtype_xcell_estimate_encoded[numeric_cols])\n",
    "val_data_feats_combined_subtype_xcell_estimate_encoded = pd.DataFrame(val_data_feats_combined_subtype_xcell_estimate_encoded, columns = val_data_feats_combined_subtype_xcell_estimate_encoded.columns)\n",
    "print(val_data_feats_combined_subtype_xcell_estimate_encoded.columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['S100B', 'NOL7', 'ILF2', 'HAT1', 'FBXO16', 'LYPLA1', 'TMEM14B', 'PDCL3',\n",
      "       'S100P', 'DKK3', 'PTPRC', 'GREM1', 'PTH2R', 'CSN3', 'SAA1', 'NOP10',\n",
      "       'TUBB6', 'B-cells', 'CD4+ T-cells', 'CD4+ Tcm', 'CD4+ Tem',\n",
      "       'CD4+ memory T-cells', 'CD4+ naive T-cells', 'CD8+ T-cells', 'CD8+ Tcm',\n",
      "       'CD8+ Tem', 'CD8+ naive T-cells', 'Memory B-cells', 'naive B-cells',\n",
      "       'pro B-cells', 'Tregs', 'ImmuneScore', 'StromaScore',\n",
      "       'MicroenvironmentScore'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "#TNBC_subtype + purity + enrichment scores - training data\n",
    "val_data_feats_106977_subtype_xcell_estimate = pd.read_table(\"/home/shayantan/TNBC-single-cell/Reproducibility_best_results/TNBC_subtype_Cell_enrichment/GSE106977_processed_data_tnbc_type_xcell_estimate.txt\")\n",
    "cols = top_feats_after_rfecv[0:17] + [\"B-cells\",\"CD4+ T-cells\",\"CD4+ Tcm\", \"CD4+ Tem\",\"CD4+ memory T-cells\", \"CD4+ naive T-cells\",\"CD8+ T-cells\",\"CD8+ Tcm\",\"CD8+ Tem\",\"CD8+ naive T-cells\",\"Memory B-cells\",\"naive B-cells\", \"pro B-cells\",\"Tregs\", \"ImmuneScore\",\"StromaScore\", \"MicroenvironmentScore\"]\n",
    "numeric_cols = top_feats_after_rfecv[0:17] + [\"B-cells\",\"CD4+ T-cells\",\"CD4+ Tcm\", \"CD4+ Tem\",\"CD4+ memory T-cells\", \"CD4+ naive T-cells\",\"CD8+ T-cells\",\"CD8+ Tcm\",\"CD8+ Tem\",\"CD8+ naive T-cells\",\"Memory B-cells\",\"naive B-cells\", \"pro B-cells\",\"Tregs\", \"ImmuneScore\",\"StromaScore\", \"MicroenvironmentScore\"]\n",
    "val_data_feats_106977_subtype_xcell_estimate = val_data_feats_106977_subtype_xcell_estimate[cols]\n",
    "#le = LabelEncoder()\n",
    "#le.fit(val_data_feats_25066_subtype_xcell_estimate['TNBC_type'])\n",
    "#val_data_feats_25066_subtype_xcell_estimate['TNBC_type'] = le.transform(val_data_feats_25066_subtype_xcell_estimate['TNBC_type'])\n",
    "#val_data_feats_25066_subtype_xcell_estimate_encoded = pd.get_dummies(val_data_feats_25066_subtype_xcell_estimate, columns=['TNBC_type'])\n",
    "#val_data_feats_25066_subtype_xcell_estimate_encoded = (val_data_feats_25066_subtype_xcell_estimate, columns=['TNBC_type'])\n",
    "sc = MinMaxScaler(); val_data_feats_106977_subtype_xcell_estimate_encoded = val_data_feats_106977_subtype_xcell_estimate\n",
    "val_data_feats_106977_subtype_xcell_estimate_encoded[numeric_cols] = sc.fit_transform(val_data_feats_106977_subtype_xcell_estimate_encoded[numeric_cols])\n",
    "val_data_feats_106977_subtype_xcell_estimate_encoded = pd.DataFrame(val_data_feats_106977_subtype_xcell_estimate_encoded, columns = val_data_feats_combined_subtype_xcell_estimate_encoded.columns)\n",
    "print(val_data_feats_106977_subtype_xcell_estimate_encoded.columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['S100B', 'NOL7', 'ILF2', 'HAT1', 'FBXO16', 'LYPLA1', 'TMEM14B', 'PDCL3',\n",
       "       'S100P', 'DKK3', 'PTPRC', 'GREM1', 'PTH2R', 'CSN3', 'SAA1', 'NOP10',\n",
       "       'TUBB6', 'B-cells', 'CD4+ T-cells', 'CD4+ Tcm', 'CD4+ Tem',\n",
       "       'CD4+ memory T-cells', 'CD4+ naive T-cells', 'CD8+ T-cells', 'CD8+ Tcm',\n",
       "       'CD8+ Tem', 'CD8+ naive T-cells', 'Memory B-cells', 'naive B-cells',\n",
       "       'pro B-cells', 'Tregs', 'ImmuneScore', 'StromaScore',\n",
       "       'MicroenvironmentScore'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_data_feats_106977_subtype_xcell_estimate_encoded.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "auroc=[];  \n",
    "df= pd.DataFrame(); \n",
    "mcc_scorer = make_scorer(matthews_corrcoef);\n",
    "cols = top_feats_after_rfecv[0:17] + ['B-cells', 'CD4+ T-cells', 'CD4+ Tcm', 'CD4+ Tem',\n",
    "       'CD4+ memory T-cells', 'CD4+ naive T-cells', 'CD8+ T-cells', 'CD8+ Tcm',\n",
    "       'CD8+ Tem', 'CD8+ naive T-cells', 'Memory B-cells', 'naive B-cells',\n",
    "       'pro B-cells', 'Tregs', 'ImmuneScore', 'StromaScore',\n",
    "       'MicroenvironmentScore']\n",
    "sampling_names = [\"RUS\", \"REDN\", \"CC\", \"IHT\", \"NearMiss\", \"EDN\", \"Tomek\", \"Allknn\",\"CondensedNN\", \"OSS\", \"SMOTE\", \"ROS\", \"ADASYN\", \"KmeansSmote\", \"BorderlineSmote\", \"SVMSmote\"]\n",
    "#sampling_names = [\"BorderlineSmote\", \"SVMSmote\"]\n",
    "sampling_techniuqes = [RepeatedEditedNearestNeighbours(), ClusterCentroids(random_state=42), InstanceHardnessThreshold(random_state=42), NearMiss(), EditedNearestNeighbours(), TomekLinks(), AllKNN(), CondensedNearestNeighbour(random_state=42), OneSidedSelection(random_state=42), SMOTE(random_state=42), RandomOverSampler(random_state=42), ADASYN(random_state=42), KMeansSMOTE(random_state=42),BorderlineSMOTE(random_state=42), SVMSMOTE(random_state=42)]\n",
    "#sampling_techniuqes = [BorderlineSMOTE(random_state=42), SVMSMOTE(random_state=42)]\n",
    "classifier_names = [\"BRF\", \"EE\", \"Bagging\", \"RF\", \"ADA\", \"ET\",\"LOGREG\",\"DT\", \"LDA\", \"KNN\", \"NB\", \"SVM\"]; \n",
    "model =[]\n",
    "#classifier_names = ['ET']\n",
    "model_index = 0\n",
    "#feats = list(range(20,28)); \n",
    "feats = [len(cols)]; auroc=[]\n",
    "\n",
    "for n in range(0,len(feats)): \n",
    "    print(\"For \",feats[n],\" genes: \") \n",
    "    #subsetiing n+1 genes and sampling technique -> REDN \n",
    "    X_train_sel = val_data_feats_25066_subtype_xcell_estimate_encoded[cols]\n",
    "    print(X_train_sel.shape)\n",
    "    scaler = [MinMaxScaler()]\n",
    "    scaler_names = [\"minmax\"]\n",
    "             \n",
    "    for s in range(0, len(scaler)):\n",
    "        print(\"Scaling scheme:\", scaler_names[s])\n",
    "        for samp in range(0,len(sampling_techniuqes)):\n",
    "            print(\"Scaling: \", scaler_names[s])\n",
    "            X_train_sel = pd.DataFrame(scaler[s].fit_transform(X_train_sel),columns = X_train_sel.columns)\n",
    "            #X_samp = X_train_sel; y_samp = train_labs\n",
    "            X_samp, y_samp=sampling_techniuqes[samp].fit_resample(X_train_sel,train_labs)\n",
    "            X_samp = pd.DataFrame(X_samp, columns = X_train_sel.columns)\n",
    "\n",
    "            for c in classifier_names:\n",
    "                name_id = sampling_names[samp]+ \"_\" + scaler_names[s]+ \"_\" + c + \"_\" + str(feats[n])\n",
    "\n",
    "                model_index = 0;  mcc_overall =[]; thres = []; auc_overall = []\n",
    "                if c == \"RF\":\n",
    "                    mod,score = model_tune_rf(X_samp, y_samp)\n",
    "                elif c == \"LOGREG\":\n",
    "                    mod,score = model_tune_logreg(X_samp, y_samp)\n",
    "                elif c == \"DT\":\n",
    "                    mod,score = model_tune_dt(X_samp, y_samp)\n",
    "                elif c == \"ET\":\n",
    "                    mod,score = model_tune_et(X_samp, y_samp)\n",
    "                elif c == \"ADA\":\n",
    "                    mod,score = model_tune_ada(X_samp, y_samp)\n",
    "                elif c == \"LDA\":\n",
    "                    mod,score = model_tune_lda(X_samp, y_samp)\n",
    "                elif c == \"KNN\":\n",
    "                    mod,score = model_tune_knn(X_samp, y_samp)\n",
    "                elif c == \"NB\":\n",
    "                    mod,score = model_tune_nb(X_samp, y_samp)\n",
    "                elif c == \"SVM\":\n",
    "                    mod,score = model_tune_svm(X_samp,y_samp)\n",
    "                elif c == \"BRF\":\n",
    "                    mod,score = model_tune_brf(X_samp,y_samp)\n",
    "                elif c == \"EE\":\n",
    "                    mod,score = model_tune_ee(X_samp, y_samp)\n",
    "                elif c == \"Bagging\":\n",
    "                    mod,score = model_tune_bagging(X_samp, y_samp)\n",
    "\n",
    "                # l=[score, c, feats[n]]\n",
    "                # row=pd.Series(l,['mcc_score','classifier','num_feats'])\n",
    "                # df=df.append([row],ignore_index=True)\n",
    "                # print(df)\n",
    "                print(\"GSE-combined\")\n",
    "                X_test=val_data_feats_combined_subtype_xcell_estimate_encoded[X_samp.columns]\n",
    "                X_test = pd.DataFrame(scaler[s].transform(X_test),columns = X_test.columns)\n",
    "                y_t, y_p, sen_c, spe_c, auprc_c, acc_c, auc_c, mcc_c, f1_c, t_c = print_metrics_with_threshold_tuning(\"GSE-combined\", X_test, val_labs_20271_20194_31519, X_samp.columns, mod)\n",
    "                l=[sen_c, spe_c, auprc_c, acc_c, auc_c, mcc_c, f1_c, \"GSE-combined\" + \"_\"+ str(t_c) + \"_\"+ name_id]\n",
    "                row=pd.Series(l,['sen','spe','auprc','acc','auc', 'mcc', 'f1', 'ID'])\n",
    "                df=df.append([row],ignore_index=True)\n",
    "                print(\"GSE106977\")\n",
    "                X_test=val_data_feats_106977_subtype_xcell_estimate_encoded[X_samp.columns]\n",
    "                X_test = pd.DataFrame(scaler[s].transform(X_test),columns = X_test.columns)\n",
    "                y_t,y_p,sen_a, spe_a, auprc_a, acc_a, auc_a, mcc_a, f1_a, t_a = print_metrics_with_threshold_tuning(\"GSE106977\", X_test, val_labs_106977, X_samp.columns, mod)\n",
    "                l=[sen_a, spe_a, auprc_a, acc_a, auc_a, mcc_a, f1_a, \"GSE106977\" + \"_\"+ str(t_a)+\"_\"+name_id]\n",
    "                row=pd.Series(l,['sen','spe','auprc','acc','auc', 'mcc', 'f1','ID'])\n",
    "                df=df.append([row],ignore_index=True)\n",
    "                df.to_csv(\"/home/shayantan/TNBC-single-cell/Reproducibility_best_results/17_gene_xcell.csv\", header=True)\n",
    "\n",
    "                print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print_metrics_with_threshold_tuning(\"ll\",val_data_feats_20271_20194_31519,val_labs_20271_20194_31519,top_feats_after_rfecv[0:17],model_gene_only[713])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Permutation test for significance using the two independent test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.851\n",
      "Name:  permute Sensitivity:  1.0 , Specificity:  0.0 AUPRC:  0.613 , AUROC:  0.623 , MCC:  0.0  , F1-weighted:  0.466\n",
      "1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.592\n",
      "Name:  permute Sensitivity:  0.904 , Specificity:  0.109 AUPRC:  0.616 , AUROC:  0.605 , MCC:  0.021  , F1-weighted:  0.517\n",
      "2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.446\n",
      "Name:  permute Sensitivity:  0.315 , Specificity:  0.783 AUPRC:  0.64 , AUROC:  0.555 , MCC:  0.106  , F1-weighted:  0.477\n",
      "3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.7000000000000001\n",
      "Name:  permute Sensitivity:  0.658 , Specificity:  0.196 AUPRC:  0.581 , AUROC:  0.403 , MCC:  -0.158  , F1-weighted:  0.46\n",
      "4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.601\n",
      "Name:  permute Sensitivity:  0.192 , Specificity:  0.717 AUPRC:  0.595 , AUROC:  0.455 , MCC:  -0.106  , F1-weighted:  0.357\n",
      "5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.661\n",
      "Name:  permute Sensitivity:  0.534 , Specificity:  0.435 AUPRC:  0.606 , AUROC:  0.392 , MCC:  -0.03  , F1-weighted:  0.501\n",
      "6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.298\n",
      "Name:  permute Sensitivity:  0.123 , Specificity:  0.913 AUPRC:  0.623 , AUROC:  0.551 , MCC:  0.057  , F1-weighted:  0.342\n",
      "7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.442\n",
      "Name:  permute Sensitivity:  0.493 , Specificity:  0.63 AUPRC:  0.646 , AUROC:  0.564 , MCC:  0.121  , F1-weighted:  0.551\n",
      "8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.634\n",
      "Name:  permute Sensitivity:  0.959 , Specificity:  0.065 AUPRC:  0.619 , AUROC:  0.567 , MCC:  0.054  , F1-weighted:  0.506\n",
      "9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.454\n",
      "Name:  permute Sensitivity:  0.411 , Specificity:  0.761 AUPRC:  0.662 , AUROC:  0.615 , MCC:  0.176  , F1-weighted:  0.541\n",
      "10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.417\n",
      "Name:  permute Sensitivity:  0.274 , Specificity:  0.826 AUPRC:  0.641 , AUROC:  0.572 , MCC:  0.115  , F1-weighted:  0.457\n",
      "11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.528\n",
      "Name:  permute Sensitivity:  0.274 , Specificity:  0.761 AUPRC:  0.622 , AUROC:  0.544 , MCC:  0.039  , F1-weighted:  0.438\n",
      "12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.736\n",
      "Name:  permute Sensitivity:  0.425 , Specificity:  0.435 AUPRC:  0.584 , AUROC:  0.404 , MCC:  -0.137  , F1-weighted:  0.436\n",
      "13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.794\n",
      "Name:  permute Sensitivity:  0.753 , Specificity:  0.109 AUPRC:  0.583 , AUROC:  0.366 , MCC:  -0.17  , F1-weighted:  0.455\n",
      "14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.484\n",
      "Name:  permute Sensitivity:  0.904 , Specificity:  0.087 AUPRC:  0.611 , AUROC:  0.438 , MCC:  -0.015  , F1-weighted:  0.502\n",
      "15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.47500000000000003\n",
      "Name:  permute Sensitivity:  0.137 , Specificity:  0.935 AUPRC:  0.635 , AUROC:  0.552 , MCC:  0.112  , F1-weighted:  0.361\n",
      "16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.463\n",
      "Name:  permute Sensitivity:  0.0 , Specificity:  0.891 AUPRC:  0.613 , AUROC:  0.491 , MCC:  -0.264  , F1-weighted:  0.198\n",
      "17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.0\n",
      "Name:  permute Sensitivity:  0.973 , Specificity:  0.0 AUPRC:  0.607 , AUROC:  0.339 , MCC:  -0.104  , F1-weighted:  0.458\n",
      "18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.457\n",
      "Name:  permute Sensitivity:  0.849 , Specificity:  0.13 AUPRC:  0.609 , AUROC:  0.44 , MCC:  -0.028  , F1-weighted:  0.508\n",
      "19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.0\n",
      "Name:  permute Sensitivity:  0.945 , Specificity:  0.022 AUPRC:  0.606 , AUROC:  0.391 , MCC:  -0.08  , F1-weighted:  0.468\n",
      "20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.47900000000000004\n",
      "Name:  permute Sensitivity:  0.685 , Specificity:  0.283 AUPRC:  0.606 , AUROC:  0.443 , MCC:  -0.034  , F1-weighted:  0.516\n",
      "21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.358\n",
      "Name:  permute Sensitivity:  0.014 , Specificity:  1.0 AUPRC:  0.619 , AUROC:  0.51 , MCC:  0.073  , F1-weighted:  0.233\n",
      "22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.253\n",
      "Name:  permute Sensitivity:  0.123 , Specificity:  0.913 AUPRC:  0.623 , AUROC:  0.624 , MCC:  0.057  , F1-weighted:  0.342\n",
      "23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.434\n",
      "Name:  permute Sensitivity:  0.425 , Specificity:  0.674 AUPRC:  0.639 , AUROC:  0.596 , MCC:  0.099  , F1-weighted:  0.521\n",
      "24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.45\n",
      "Name:  permute Sensitivity:  0.0 , Specificity:  1.0 AUPRC:  0.613 , AUROC:  0.455 , MCC:  0.0  , F1-weighted:  0.216\n",
      "25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.419\n",
      "Name:  permute Sensitivity:  0.027 , Specificity:  0.978 AUPRC:  0.615 , AUROC:  0.588 , MCC:  0.018  , F1-weighted:  0.247\n",
      "26\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.42\n",
      "Name:  permute Sensitivity:  0.0 , Specificity:  0.978 AUPRC:  0.613 , AUROC:  0.425 , MCC:  -0.116  , F1-weighted:  0.212\n",
      "27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.751\n",
      "Name:  permute Sensitivity:  0.89 , Specificity:  0.109 AUPRC:  0.613 , AUROC:  0.51 , MCC:  -0.001  , F1-weighted:  0.511\n",
      "28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.506\n",
      "Name:  permute Sensitivity:  0.822 , Specificity:  0.174 AUPRC:  0.612 , AUROC:  0.499 , MCC:  -0.005  , F1-weighted:  0.523\n",
      "29\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.401\n",
      "Name:  permute Sensitivity:  0.411 , Specificity:  0.63 AUPRC:  0.624 , AUROC:  0.558 , MCC:  0.041  , F1-weighted:  0.497\n",
      "30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.309\n",
      "Name:  permute Sensitivity:  0.0 , Specificity:  1.0 AUPRC:  0.613 , AUROC:  0.483 , MCC:  0.0  , F1-weighted:  0.216\n",
      "31\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.497\n",
      "Name:  permute Sensitivity:  0.795 , Specificity:  0.348 AUPRC:  0.65 , AUROC:  0.554 , MCC:  0.158  , F1-weighted:  0.603\n",
      "32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.621\n",
      "Name:  permute Sensitivity:  0.534 , Specificity:  0.348 AUPRC:  0.588 , AUROC:  0.453 , MCC:  -0.116  , F1-weighted:  0.466\n",
      "33\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.531\n",
      "Name:  permute Sensitivity:  0.795 , Specificity:  0.37 AUPRC:  0.656 , AUROC:  0.625 , MCC:  0.18  , F1-weighted:  0.613\n",
      "34\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.397\n",
      "Name:  permute Sensitivity:  0.178 , Specificity:  0.739 AUPRC:  0.597 , AUROC:  0.449 , MCC:  -0.099  , F1-weighted:  0.351\n",
      "35\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.41600000000000004\n",
      "Name:  permute Sensitivity:  0.192 , Specificity:  0.826 AUPRC:  0.618 , AUROC:  0.495 , MCC:  0.022  , F1-weighted:  0.386\n",
      "36\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.486\n",
      "Name:  permute Sensitivity:  0.137 , Specificity:  0.891 AUPRC:  0.621 , AUROC:  0.475 , MCC:  0.042  , F1-weighted:  0.351\n",
      "37\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.502\n",
      "Name:  permute Sensitivity:  0.164 , Specificity:  0.913 AUPRC:  0.636 , AUROC:  0.492 , MCC:  0.111  , F1-weighted:  0.383\n",
      "38\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.47500000000000003\n",
      "Name:  permute Sensitivity:  0.288 , Specificity:  0.739 AUPRC:  0.62 , AUROC:  0.527 , MCC:  0.029  , F1-weighted:  0.442\n",
      "39\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.28200000000000003\n",
      "Name:  permute Sensitivity:  0.055 , Specificity:  0.978 AUPRC:  0.624 , AUROC:  0.567 , MCC:  0.08  , F1-weighted:  0.28\n",
      "40\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.65\n",
      "Name:  permute Sensitivity:  0.685 , Specificity:  0.239 AUPRC:  0.596 , AUROC:  0.479 , MCC:  -0.082  , F1-weighted:  0.495\n",
      "41\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.225\n",
      "Name:  permute Sensitivity:  0.0 , Specificity:  1.0 AUPRC:  0.613 , AUROC:  0.439 , MCC:  0.0  , F1-weighted:  0.216\n",
      "42\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.555\n",
      "Name:  permute Sensitivity:  0.699 , Specificity:  0.217 AUPRC:  0.594 , AUROC:  0.525 , MCC:  -0.092  , F1-weighted:  0.49\n",
      "43\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.41300000000000003\n",
      "Name:  permute Sensitivity:  0.603 , Specificity:  0.391 AUPRC:  0.612 , AUROC:  0.533 , MCC:  -0.006  , F1-weighted:  0.522\n",
      "44\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.514\n",
      "Name:  permute Sensitivity:  0.849 , Specificity:  0.217 AUPRC:  0.63 , AUROC:  0.562 , MCC:  0.085  , F1-weighted:  0.56\n",
      "45\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.342\n",
      "Name:  permute Sensitivity:  0.0 , Specificity:  1.0 AUPRC:  0.613 , AUROC:  0.554 , MCC:  0.0  , F1-weighted:  0.216\n",
      "46\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.531\n",
      "Name:  permute Sensitivity:  0.959 , Specificity:  0.065 AUPRC:  0.619 , AUROC:  0.593 , MCC:  0.054  , F1-weighted:  0.506\n",
      "47\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.522\n",
      "Name:  permute Sensitivity:  0.548 , Specificity:  0.63 AUPRC:  0.662 , AUROC:  0.566 , MCC:  0.174  , F1-weighted:  0.585\n",
      "48\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.457\n",
      "Name:  permute Sensitivity:  0.425 , Specificity:  0.63 AUPRC:  0.627 , AUROC:  0.557 , MCC:  0.055  , F1-weighted:  0.506\n",
      "49\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.5730000000000001\n",
      "Name:  permute Sensitivity:  0.781 , Specificity:  0.239 AUPRC:  0.618 , AUROC:  0.563 , MCC:  0.023  , F1-weighted:  0.54\n",
      "50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.613\n",
      "Name:  permute Sensitivity:  0.397 , Specificity:  0.5 AUPRC:  0.591 , AUROC:  0.452 , MCC:  -0.101  , F1-weighted:  0.442\n",
      "51\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.449\n",
      "Name:  permute Sensitivity:  0.123 , Specificity:  0.935 AUPRC:  0.63 , AUROC:  0.558 , MCC:  0.094  , F1-weighted:  0.347\n",
      "52\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.525\n",
      "Name:  permute Sensitivity:  0.534 , Specificity:  0.63 AUPRC:  0.658 , AUROC:  0.561 , MCC:  0.161  , F1-weighted:  0.577\n",
      "53\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.374\n",
      "Name:  permute Sensitivity:  0.288 , Specificity:  0.522 AUPRC:  0.577 , AUROC:  0.5 , MCC:  -0.193  , F1-weighted:  0.374\n",
      "54\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.512\n",
      "Name:  permute Sensitivity:  0.808 , Specificity:  0.239 AUPRC:  0.625 , AUROC:  0.556 , MCC:  0.057  , F1-weighted:  0.553\n",
      "55\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.442\n",
      "Name:  permute Sensitivity:  0.0 , Specificity:  1.0 AUPRC:  0.613 , AUROC:  0.606 , MCC:  0.0  , F1-weighted:  0.216\n",
      "56\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.641\n",
      "Name:  permute Sensitivity:  0.781 , Specificity:  0.13 AUPRC:  0.593 , AUROC:  0.396 , MCC:  -0.111  , F1-weighted:  0.48\n",
      "57\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.5750000000000001\n",
      "Name:  permute Sensitivity:  0.479 , Specificity:  0.5 AUPRC:  0.609 , AUROC:  0.465 , MCC:  -0.02  , F1-weighted:  0.494\n",
      "58\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.592\n",
      "Name:  permute Sensitivity:  1.0 , Specificity:  0.0 AUPRC:  0.613 , AUROC:  0.456 , MCC:  0.0  , F1-weighted:  0.466\n",
      "59\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.223\n",
      "Name:  permute Sensitivity:  0.671 , Specificity:  0.261 AUPRC:  0.598 , AUROC:  0.4 , MCC:  -0.072  , F1-weighted:  0.499\n",
      "60\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.28200000000000003\n",
      "Name:  permute Sensitivity:  0.164 , Specificity:  0.891 AUPRC:  0.629 , AUROC:  0.554 , MCC:  0.077  , F1-weighted:  0.378\n",
      "61\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.398\n",
      "Name:  permute Sensitivity:  0.836 , Specificity:  0.13 AUPRC:  0.606 , AUROC:  0.377 , MCC:  -0.046  , F1-weighted:  0.503\n",
      "62\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.6890000000000001\n",
      "Name:  permute Sensitivity:  0.89 , Specificity:  0.043 AUPRC:  0.598 , AUROC:  0.444 , MCC:  -0.116  , F1-weighted:  0.466\n",
      "63\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.671\n",
      "Name:  permute Sensitivity:  0.959 , Specificity:  0.0 AUPRC:  0.604 , AUROC:  0.51 , MCC:  -0.128  , F1-weighted:  0.454\n",
      "64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.634\n",
      "Name:  permute Sensitivity:  0.178 , Specificity:  0.87 AUPRC:  0.626 , AUROC:  0.449 , MCC:  0.063  , F1-weighted:  0.385\n",
      "65\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.201\n",
      "Name:  permute Sensitivity:  0.041 , Specificity:  1.0 AUPRC:  0.629 , AUROC:  0.618 , MCC:  0.128  , F1-weighted:  0.268\n",
      "66\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.432\n",
      "Name:  permute Sensitivity:  0.315 , Specificity:  0.478 AUPRC:  0.574 , AUROC:  0.382 , MCC:  -0.206  , F1-weighted:  0.379\n",
      "67\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.501\n",
      "Name:  permute Sensitivity:  0.397 , Specificity:  0.717 AUPRC:  0.644 , AUROC:  0.494 , MCC:  0.117  , F1-weighted:  0.517\n",
      "68\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.379\n",
      "Name:  permute Sensitivity:  0.11 , Specificity:  1.0 AUPRC:  0.656 , AUROC:  0.554 , MCC:  0.213  , F1-weighted:  0.348\n",
      "69\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.315\n",
      "Name:  permute Sensitivity:  0.589 , Specificity:  0.413 AUPRC:  0.614 , AUROC:  0.476 , MCC:  0.002  , F1-weighted:  0.524\n",
      "70\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.501\n",
      "Name:  permute Sensitivity:  0.164 , Specificity:  0.935 AUPRC:  0.644 , AUROC:  0.473 , MCC:  0.145  , F1-weighted:  0.389\n",
      "71\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.421\n",
      "Name:  permute Sensitivity:  0.479 , Specificity:  0.5 AUPRC:  0.609 , AUROC:  0.519 , MCC:  -0.02  , F1-weighted:  0.494\n",
      "72\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.543\n",
      "Name:  permute Sensitivity:  1.0 , Specificity:  0.0 AUPRC:  0.613 , AUROC:  0.446 , MCC:  0.0  , F1-weighted:  0.466\n",
      "73\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.51\n",
      "Name:  permute Sensitivity:  0.658 , Specificity:  0.304 AUPRC:  0.605 , AUROC:  0.494 , MCC:  -0.04  , F1-weighted:  0.512\n",
      "74\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.425\n",
      "Name:  permute Sensitivity:  0.342 , Specificity:  0.63 AUPRC:  0.607 , AUROC:  0.495 , MCC:  -0.028  , F1-weighted:  0.449\n",
      "75\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.5750000000000001\n",
      "Name:  permute Sensitivity:  0.877 , Specificity:  0.109 AUPRC:  0.61 , AUROC:  0.483 , MCC:  -0.022  , F1-weighted:  0.506\n",
      "76\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.397\n",
      "Name:  permute Sensitivity:  0.041 , Specificity:  1.0 AUPRC:  0.629 , AUROC:  0.559 , MCC:  0.128  , F1-weighted:  0.268\n",
      "77\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.582\n",
      "Name:  permute Sensitivity:  0.507 , Specificity:  0.522 AUPRC:  0.62 , AUROC:  0.493 , MCC:  0.028  , F1-weighted:  0.519\n",
      "78\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.316\n",
      "Name:  permute Sensitivity:  0.068 , Specificity:  0.935 AUPRC:  0.614 , AUROC:  0.621 , MCC:  0.006  , F1-weighted:  0.287\n",
      "79\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.5680000000000001\n",
      "Name:  permute Sensitivity:  0.904 , Specificity:  0.109 AUPRC:  0.616 , AUROC:  0.545 , MCC:  0.021  , F1-weighted:  0.517\n",
      "80\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.307\n",
      "Name:  permute Sensitivity:  0.014 , Specificity:  1.0 AUPRC:  0.619 , AUROC:  0.593 , MCC:  0.073  , F1-weighted:  0.233\n",
      "81\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.454\n",
      "Name:  permute Sensitivity:  0.137 , Specificity:  0.957 AUPRC:  0.644 , AUROC:  0.591 , MCC:  0.151  , F1-weighted:  0.367\n",
      "82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.66\n",
      "Name:  permute Sensitivity:  0.753 , Specificity:  0.152 AUPRC:  0.592 , AUROC:  0.468 , MCC:  -0.113  , F1-weighted:  0.48\n",
      "83\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.0\n",
      "Name:  permute Sensitivity:  0.178 , Specificity:  0.674 AUPRC:  0.587 , AUROC:  0.432 , MCC:  -0.17  , F1-weighted:  0.333\n",
      "84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.431\n",
      "Name:  permute Sensitivity:  0.11 , Specificity:  0.957 AUPRC:  0.634 , AUROC:  0.499 , MCC:  0.116  , F1-weighted:  0.338\n",
      "85\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.481\n",
      "Name:  permute Sensitivity:  0.685 , Specificity:  0.457 AUPRC:  0.65 , AUROC:  0.566 , MCC:  0.143  , F1-weighted:  0.595\n",
      "86\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.596\n",
      "Name:  permute Sensitivity:  0.712 , Specificity:  0.174 AUPRC:  0.588 , AUROC:  0.483 , MCC:  -0.129  , F1-weighted:  0.474\n",
      "87\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.495\n",
      "Name:  permute Sensitivity:  0.808 , Specificity:  0.239 AUPRC:  0.625 , AUROC:  0.478 , MCC:  0.057  , F1-weighted:  0.553\n",
      "88\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.63\n",
      "Name:  permute Sensitivity:  0.575 , Specificity:  0.304 AUPRC:  0.587 , AUROC:  0.459 , MCC:  -0.121  , F1-weighted:  0.469\n",
      "89\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.403\n",
      "Name:  permute Sensitivity:  0.699 , Specificity:  0.239 AUPRC:  0.599 , AUROC:  0.459 , MCC:  -0.068  , F1-weighted:  0.501\n",
      "90\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.521\n",
      "Name:  permute Sensitivity:  0.89 , Specificity:  0.087 AUPRC:  0.608 , AUROC:  0.5 , MCC:  -0.037  , F1-weighted:  0.496\n",
      "91\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.491\n",
      "Name:  permute Sensitivity:  0.247 , Specificity:  0.848 AUPRC:  0.64 , AUROC:  0.48 , MCC:  0.113  , F1-weighted:  0.441\n",
      "92\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.47700000000000004\n",
      "Name:  permute Sensitivity:  0.863 , Specificity:  0.196 AUPRC:  0.628 , AUROC:  0.468 , MCC:  0.078  , F1-weighted:  0.554\n",
      "93\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.0\n",
      "Name:  permute Sensitivity:  0.986 , Specificity:  0.0 AUPRC:  0.61 , AUROC:  0.426 , MCC:  -0.073  , F1-weighted:  0.462\n",
      "94\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.35100000000000003\n",
      "Name:  permute Sensitivity:  0.11 , Specificity:  0.848 AUPRC:  0.605 , AUROC:  0.475 , MCC:  -0.062  , F1-weighted:  0.313\n",
      "95\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.801\n",
      "Name:  permute Sensitivity:  0.479 , Specificity:  0.522 AUPRC:  0.614 , AUROC:  0.456 , MCC:  0.001  , F1-weighted:  0.502\n",
      "96\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.684\n",
      "Name:  permute Sensitivity:  1.0 , Specificity:  0.0 AUPRC:  0.613 , AUROC:  0.553 , MCC:  0.0  , F1-weighted:  0.466\n",
      "97\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.758\n",
      "Name:  permute Sensitivity:  1.0 , Specificity:  0.0 AUPRC:  0.613 , AUROC:  0.487 , MCC:  0.0  , F1-weighted:  0.466\n",
      "98\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.507\n",
      "Name:  permute Sensitivity:  0.534 , Specificity:  0.739 AUPRC:  0.694 , AUROC:  0.657 , MCC:  0.269  , F1-weighted:  0.616\n",
      "99\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1347127/1401251787.py:17: UserWarning: you are shuffling a 'Series' object which is not a subclass of 'Sequence'; `shuffle` is not guaranteed to behave correctly. E.g., non-numpy array/tensor objects with view semantics may contain duplicates after shuffling.\n",
      "  np.random.shuffle(y_samp)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tuning Done\n",
      "Threshold:  0.395\n",
      "Name:  permute Sensitivity:  0.233 , Specificity:  0.848 AUPRC:  0.636 , AUROC:  0.538 , MCC:  0.098  , F1-weighted:  0.429\n"
     ]
    }
   ],
   "source": [
    "#permutation test using the same sampling-classifier-num_feats configuration as the best results - ROS-ET-0.5\n",
    "feats = [20]; sc = MinMaxScaler();\n",
    "X_train_sel = train_data_feats[top_feats_after_rfecv[0:17]]\n",
    "X_train_sel = pd.DataFrame(sc.fit_transform(X_train_sel),columns = X_train_sel.columns)\n",
    "samp = RandomOverSampler(random_state=42)\n",
    "X_samp, y_samp=samp.fit_resample(X_train_sel,train_labs)\n",
    "X_samp = pd.DataFrame(X_samp, columns = X_train_sel.columns)\n",
    "X_test = val_data_feats_106977[top_feats_after_rfecv[0:17]]; y_test = val_labs_106977\n",
    "\n",
    "\n",
    "# Permutation test or the resampling test.\n",
    "permuted_accuracies = []; p_sen=[];p_spe=[];p_auroc=[];p_auprc=[];p_mcc=[];p_f1=[]\n",
    "permutation_iters = 100 # We run the procedure many times to have a 'good enough' estimate of the accuracy of a chance model.\n",
    "for i in range(permutation_iters):\n",
    "    print(i)\n",
    "    # Shuffle the dataset here so that the target species/labels are `randomly` assigned to the samples.\n",
    "    np.random.shuffle(y_samp)\n",
    "    \n",
    "    # When we obtain train and test splits, the assignments are random.\n",
    "    best_mod, b = model_tune_et(X_samp,y_samp)\n",
    "    y_t, y_p, sen_c, spe_c, auprc_c, acc_c, auc_c, mcc_c, f1_c, t_c = print_metrics_with_threshold_tuning(\"permute\",X_test,y_test,top_feats_after_rfecv[0:17],best_mod)\n",
    "    p_sen.append(sen_c)\n",
    "    p_spe.append(spe_c)\n",
    "    p_auroc.append(auc_c)\n",
    "    p_auprc.append(auprc_c)\n",
    "    p_mcc.append(mcc_c)\n",
    "    p_f1.append(f1_c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_results_106977 = pd.DataFrame({'Sen': p_sen,'Spe': p_spe, 'AUC': p_auroc,'AUPRC': p_auprc,'MCC':p_mcc,'F1':p_f1})\n",
    "random_results_106977.to_csv('/home/shayantan/TNBC-single-cell/Reproducibility_best_results/random_permutation_test_results_106977.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_results_combined = pd.DataFrame({'Sen': p_sen,'Spe': p_spe, 'AUC': p_auroc,'AUPRC': p_auprc,'MCC':p_mcc,'F1':p_f1})\n",
    "random_results_combined.to_csv('/home/shayantan/TNBC-single-cell/Reproducibility_best_results/random_permutation_test_results_20271-20194-31519.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy import stats\n",
    "\n",
    "def test_significantly_less(data, fixed_value, alpha=0.05):\n",
    "    \"\"\"\n",
    "    Perform one-sample t-test to determine if the data is significantly less than the fixed value.\n",
    "    \n",
    "    Parameters:\n",
    "    - data: list or array-like, the data values\n",
    "    - fixed_value: float, the fixed value to compare against\n",
    "    - alpha: float, the significance level (default is 0.05)\n",
    "    \n",
    "    Returns:\n",
    "    - result: tuple, containing test statistic and p-value\n",
    "    - is_significant: bool, indicating if the difference is statistically significant\n",
    "    \"\"\"\n",
    "    # Perform one-sample t-test\n",
    "    result = stats.ttest_1samp(data, fixed_value)\n",
    "    \n",
    "    # Extract test statistic and p-value\n",
    "    t_statistic, p_value = result\n",
    "    \n",
    "    # Check if the p-value is less than alpha\n",
    "    is_significant = p_value < alpha\n",
    "    \n",
    "    return result, is_significant\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for permutation tests - t test p value - 20271-20194-31519\n",
      "\n",
      "Test statistic: -42.8637003433758\n",
      "P-value: 9.840616791303992e-66\n",
      "Is significantly less than fixed value at 0.05 level? True\n"
     ]
    }
   ],
   "source": [
    "# Example usage:\n",
    "print(\"Results for permutation tests - t test p value - 20271-20194-31519\\n\")\n",
    "data = p_auroc  # Sample data\n",
    "fixed_value = 0.782  # Fixed value to compare against\n",
    "alpha = 0.05  # Significance level\n",
    "\n",
    "result, is_significant = test_significantly_less(data, fixed_value, alpha)\n",
    "print(\"Test statistic:\", result[0])\n",
    "print(\"P-value:\", result[1])\n",
    "print(\"Is significantly less than fixed value at\", alpha, \"level?\", is_significant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Results for permutation tests - t test p value - 106977\n",
      "\n",
      "Test statistic: -23.46188652872625\n",
      "P-value: 3.167838675140307e-42\n",
      "Is significantly less than fixed value at 0.05 level? True\n"
     ]
    }
   ],
   "source": [
    "# Example usage:\n",
    "print(\"Results for permutation tests - t test p value - 106977\\n\")\n",
    "data = p_auroc  # Sample data\n",
    "fixed_value = 0.666  # Fixed value to compare against\n",
    "alpha = 0.05  # Significance level\n",
    "\n",
    "result, is_significant = test_significantly_less(data, fixed_value, alpha)\n",
    "print(\"Test statistic:\", result[0])\n",
    "print(\"P-value:\", result[1])\n",
    "print(\"Is significantly less than fixed value at\", alpha, \"level?\", is_significant)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For 20271-20194-31519 results of permutation tests: Sen:  0.49858 Spe:  0.49501000000000006 AUROC:  0.49305 AUPRC:  0.6192 MCC:  -0.005390000000000001 F1:  0.4993699999999999\n"
     ]
    }
   ],
   "source": [
    "print(\"For 20271-20194-31519 results of permutation tests:\", \"Sen: \", np.mean(p_sen), \"Spe: \", np.mean(p_spe), \"AUROC: \", np.mean(p_auroc), \"AUPRC: \", np.mean(p_auprc), \"MCC: \", np.mean(p_mcc), \"F1: \", np.mean(p_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For 106977 results of permutation tests: Sen:  0.4881999999999999 Spe:  0.5186799999999999 AUROC:  0.5056999999999999 AUPRC:  0.61736 MCC:  0.009219999999999999 F1:  0.43709000000000003\n"
     ]
    }
   ],
   "source": [
    "print(\"For 106977 results of permutation tests:\", \"Sen: \", np.mean(p_sen), \"Spe: \", np.mean(p_spe), \"AUROC: \", np.mean(p_auroc), \"AUPRC: \", np.mean(p_auprc), \"MCC: \", np.mean(p_mcc), \"F1: \", np.mean(p_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAewAAAGeCAYAAABWwDKqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA1UUlEQVR4nO3deVxVdeL/8fdluZi4L6GoOY5laqQlNY5mYqjhvmSOuA02lVMuzWjpz3JJyywrM7WcaZoy09QvKuWOSmpZqSmZhlvJuOKOiLIkyL2/PxxvOQgHiXMPB17Px2MeI3DO+bz5gL09n3vuOQ632+0WAAAo1nysDgAAAIxR2AAA2ACFDQCADVDYAADYAIUNAIANUNgAANgAhQ0ApdyePXsUFRWlzp07KyIiQn369NGOHTusjmWq06dPq0uXLobbRUVFac+ePYUeZ+DAgVq2bFmh9/81vyI5CgDAltxut/7617/q5Zdf1kMPPSRJWr16tYYOHapNmzbplltu+U3Hz8nJka+vb1FELTI5OTkKCgrSypUrDbedO3euFxIVDGfYAFCKXbhwQWfPnlXTpk09n+vUqZOWL1/uKeu5c+eqU6dOat++vaZMmaJr99v6+OOP1bFjR0VERGjIkCE6f/68JGnMmDF65ZVX1LlzZ8XGxiorK0uTJ09WRESEOnXqpHfffVc3umfX5cuXNWHCBEVERKhDhw6aOnWqcnJyJElhYWGKjo7Wo48+qhYtWmjKlCk3/H5OnDihxx9/3DPWZ599Jkk6fvy4HnjgAb3yyivq37+/jh8/rsaNG0uSfv75Z40cOVLh4eF64oknNG3aNI0aNUqSFB4erh07dujo0aNq1aqV5s+fry5duqhVq1aewne73ZoyZYoefvhhhYeHa9SoUcrOzv6tP5pcKGwAKMUqV66se+65R1FRUVq8eLGOHz8uSQoKCpIk7dq1S/Pnz1d0dLRWrVql7777TmvWrNH333+vDz74QJ988onWrl2r4OBgvfnmm57jbtu2TUuXLlXnzp01f/58HTp0SCtXrtSSJUsUFxenL774IleWuXPn6tSpU1q9erWWL1+uHTt2aMWKFZIkX19fbdu2TdHR0frss8+0YMECnTx5Mtcxxo8fr+bNm2vt2rX697//rVdeeUXHjh2TJKWmpqpRo0ZatGjRdfssXbpUJ06c0Lp16zR58mR99tlnuVYFfHx8lJKSIofDoZUrV2rs2LF6++23JUmff/65vv76a61cuVKxsbHas2ePVq9eXcifSN4obAAo5T788ENFRETo448/Vtu2bdW5c2etW7dOkrRx40aFhYWpXLlycjqdmjdvniIiIrRp0ya1bdtWVapUkST16NFDX3/9teeYLVq0UJkyZSRdLbRevXrJ399fZcuWVffu3bV+/fpcOTZt2qRHHnlEvr6+cjqd6tix43XH7Nq1q3x8fBQUFKRq1arp1KlT1+2fnZ2tb775Ro8++qgkKTg4WKGhodq6davn6w8//HCucXfs2KGIiAj5+fmpRo0aatGixQ3n6cqVK+rZs6ckKSQkxDN+u3btFBMTI6fTKafTqbvvvtvzj4SixGvYAFDKBQYGatiwYRo2bJjOnTunmJgYjRw5UsuWLdP58+dVvXp1z7bXlsnPnz+vatWqeT5fsWJFz5L4tY+vSU5O1uuvv66ZM2dKkrKysq5bgv/1dpUqVcrzmOXKlfP82cfHx7Ncfk1KSopcLpcqV6583TFSUlIkXT1L//UxrklNTb0ub82aNXX69Olc2/n6+qps2bKSJIfDIZfLJenqBWxTpkzRgQMHJEnnzp1TVFRUrv1/KwobAEqxU6dOKSkpSaGhoZKkatWqafDgwYqNjdXBgwdVpUoVT+FJ8vz5Rp//dYH/WvXq1TVo0CC1bds23yxVq1Yt8DFvpHLlyp6l62tn/ikpKapatWq++5UrV05paWmej29U1vl5++235evrq+XLl8vpdOq55567qf0LiiVxACjFTp48qaefflq7d+/2fC4hIUEnT55USEiIwsPDtWHDBl24cEFXrlzR0KFDtXnzZoWHh2vjxo2egl2yZInCwsJuOEZ4eLgWL16s7Oxsud1uzZ49W5s3b8613UMPPaRly5bJ5XIpMzNTq1atyvOYN+Lv769WrVpp6dKlkqRjx47p+++/V8uWLfPdLyQkRBs2bJDL5VJSUtINs+Xn4sWLatiwoZxOp3bv3q3vvvtO6enpN3WMguAMGwBKsXvvvVcvv/yyJk2apLS0NPn5+alSpUp66623VKtWLdWqVUt/+ctfFBkZqZycHIWFhalr165yOBx6/PHH1bdvX7ndbjVu3FgTJ0684Rj9+/dXUlKSunbtqpycHDVp0uSGS8Z//vOfdezYMXXs2FHS1avVr/25oCZNmqRx48ZpyZIl8vf318svv6yaNWt6Lqa7kcjISG3btk3h4eFq1KiRunTpoosXLxZ4zMcee0yjR49WdHS0WrRooeeff15jxoxRs2bNbiq7EQfPwwYAlHZut1sOh0OSNHXqVLlcLj3//PMWp7oeS+IAgFJt48aN6tWrl7KyspSenq5Nmzbp3nvvtTpWLiyJAwBKtdatW+uLL75Qx44d5ePjo7Zt2yoiIsLqWLmwJA4AgA2wJA4AgA1Q2AAAjx07dig8PNxwu127dmn//v2/aaykpCQNGDBAYWFh6tixo7Zt25ZrmxMnTqhDhw7X/e+ee+7Rhg0bJEnvvfee597jTz311E2/h9pOKGwAwE1bunSp585ehTVu3DjP68evvPKKRowYoczMzOu2CQ4OVmxsrOd/c+bMUY0aNdSyZUtt3rxZS5cu1dKlSxUbG6vbb79dr7322m/KVJxR2ABQys2ePVthYWHq0aOHvvnmG8/nL1++rOeee04REREKDw/X1KlTJUkLFy7UsmXL9MYbb2jOnDn5Pq1q/fr1N3x71MWLF7V9+3YNGDBAktSsWTPVqlXrhmfZv/bmm29qyJAhKlOmjA4ePKjGjRt7bjd6//33KzExsUjmpDiisAGgFDt48KA++ugjLV26VDExMdedNS9cuFApKSlas2aNPv30U8XExGjHjh3q27evmjRpolGjRumxxx7L92lV7du316uvvppr3KNHj6py5cqee3NLUp06dXT48OE8sx44cED79+9Xly5dJF0t6O+//16nT5+Wy+XS559/bnhXMzvjbV0AUIpt375doaGhnnt2d+vWTfv27ZMkDRo0SP369ZOPj48qVqyoO+64Q8ePH9d999133THatWunBx98UE6nU5IK9LSqn3/+2bP9NQEBAfne0nPevHkaMGCAfHyunmuGhISoe/fuCg8PV2BgoKpVq6ZPPvnk5ibARjjDBoBS7H+fVPXrPycmJmrYsGGei7oSEhI8T6j6tdOnT2v06NGei8I+//xzGb1j+JZbbrnugRuSlJaWpsDAwBtuf/nyZa1atUrt27f3fC4uLk5ffvmlNm/erG3btqlz584aPXp0gb5vO6KwAaAUq1Chgi5duuT5+NePs5w0aZJuv/12rVmzRrGxsWrUqNENj/Hrp1XFxsaqTZs2huPWrVtX6enpunDhgudzhw8f1u9///sbbr9jxw7deeed1z296+uvv1br1q1VpUoVORwOderUSdu3bzcc264obAAoxZo1a6YdO3bo/PnzysnJ0fLlyz1fu3jxou666y75+Pho48aNOnLkiGfJ2s/Pz1P0hXlaVbly5dSqVSvPEva2bduUkpKi5s2b33D73bt3q06dOtd97vbbb9fWrVv1888/S5K++uorNWjQoHATYQO8hg0ApVjDhg3Vt29f9ezZUxUqVFDXrl09r2E//fTTevnllzVz5kx17NhRQ4cO1cyZMxUSEqJ27drpzTffVFJSUr5Pq3I4HNqwYcMNLzybNGmSRo0apejoaJUvX16zZs1SQECAJKlDhw6aP3++54z67NmzuvXWW6/bv0+fPjp8+LC6desmHx8fVa9e/YbjlBTcmhQAABtgSRwAABugsAEAsAEKGwAAG6CwAQCwAQobAKBPPvlE/+///T/Tx7nzzjt16tQpU479zTffqEOHDmrTpo0GDhyY5zh5bed2u/X666+rdevWCgsL07Rp0zz75PVksWeffVbR0dGmfD+5uAEApdrRo0fdDz74oPvSpUumj9WgQQP3yZMni/y4aWlp7ubNm7t37Njhdrvd7vfee8/9xBNP3NR2y5Ytc/fs2dOdkZHhzszMdPfo0cO9atUqt9vtdg8aNMj93nvvud1utzs+Pt7dokULd0ZGhjs1NdXdqlUr94kTJ4r8e/pfnGEDQCn3/vvvq1evXipXrpy2bNmiLl26aOrUqerQoYM6deqk3bt359pn06ZN6tq163Wf69atm7788kulpKRo8ODBioiIUNu2bTVnzpxc+8fExGjQoEE3/DgrK0uTJ09WRESEOnXqpHfffddzq9Np06Zp4cKFuY63detW1a1bV6GhoZKk/v37a8uWLbluf5rfduvWrdOf/vQn3XLLLSpTpox69+6t2NjYfJ8sVqFCBfXs2VMffvhhAWe78ChsACjF3G634uLiPPfo9vHx0X/+8x/dd999io2N1YABAzRp0qRc+7Vs2VInT570POTj2LFjOn36tFq0aKHZs2fr1ltv1dq1a/XRRx9p2rRpOnnyZIEzzZ8/X4cOHdLKlSu1ZMkSxcXF6YsvvpB0dQm6b9++ufY5cuSIateu7fk4MDBQlSpV0tGjRwu83ZEjR667m9q1p4cZPVmsXbt2WrduXYG/v8KisAGgFDt+/Ljn1qLXlC1bVm3btpV09Y5je/bs8Tzf+hqn06mHHnpIGzZskHT1QRzt2rWTv7+/XnjhBU2YMEHS1WKrXr26jh8/XuBMn3/+uXr16iV/f3+VLVtW3bt31/r16/PdJzMzM9fTv5xOpzIyMgq83f9+LSAgQBkZGYZPFrvrrrt09uxZnT59usDfY2Fwa1IAKMVSUlJUuXJlzyMrpasPBLmmfPnycrvdunTpkkaMGOEppdjYWEVERGjevHmKiopSXFychgwZIknauXOnpk+frjNnzsjhcOjs2bM3fMpXXpKTk/X6669r5syZkq4ukTdt2jTffcqWLZvr/uXp6enXnRUbbfe/TxBLS0u74eevfe3ak8V8fX1VsWJFJScnKygoqMDf582isAEA10lNTZXb7ZbD4dClS5fkcDhUoUIFzZ0797rtWrdurbFjx+rw4cM6fPiw58Edo0aN0mOPPeZ5zbd169a5xvDx8bnuEZy/LtHq1atr0KBBnrP8gqhXr54+/fRTz8fnz59Xenq6brvttgJvV69ePR05csTztSNHjqh+/frXPVmsUqVKkq4+Wax3797XHdtt8p2+WRIHgFKscuXKunDhwnVlk5GRobi4OElXz6Tvuece+fnlPr9zOp164IEH9MYbb6ht27aebS5evKiQkBBJUnR0tDIyMnKd1QYFBenYsWPKzs7WlStXPONJUnh4uBYvXqzs7Gy53W7Nnj1bmzdvzvf7+OMf/6jk5GTP4zUXLFigsLAwlStXrsDbde7cWUuXLlVGRoYyMjIUExOjzp07Gz5ZLCcnR6mpqapatarBbP82nGEDQClWu3ZtlStXTgcOHPC8jl27dm19++23evPNN+Xv76/XX389z/07dOig4cOH66OPPvJ87plnntFTTz2lKlWqqH///urbt68mTJigO+64w7NN8+bNFRISoh49eqh27dpq3ry5tmzZIunqldtJSUnq2rWrcnJy1KRJE0VFRUm6epV4cHBwrgvPypQpoxkzZmjChAlKT09X/fr1NXXqVElXH805Y8YMffDBB/luFxERoX379qljx46SpN69e6tdu3aS8n+y2J49e1S1alXVqFGj0D+HguBpXQBQyo0fP15BQUEaNmyYtm3bpnHjxhle5IVfTJ8+XZcuXfJcaGcWlsQBoJQbPHiwlixZkuvCKhi7dOmSYmJi9Pjjj5s+FoUNAKVcnTp19OSTT2ry5MlWR7GdiRMnaujQoapVq5bpY7EkDgCADXCGDQCADVDYAADYAIUNAIANUNgAANgAhQ0AgA1Q2AAA2ACFDQCADVDYAADYAIUNAIANUNgAANgAhQ0AgA1Q2AAA2ACFDQCADVDYAADYAIUNAIANUNgAANgAhQ0AgA1Q2AAA2ACFDQCADVDYAADYAIUNAIANUNgAANgAhQ0AgA1Q2AAA2ACFDQCADfiZdWCXy6WJEyfqp59+ko+Pj1566SXVr1/frOEAACjRTDvDjouL06VLl7Rw4UKNHz9eU6ZMMWsoAABKPNMK+/DhwwoJCZEkNWzYUAcOHFBOTo5ZwwEAUKKZtiTeoEEDzZs3T4MGDdK+ffuUnJyslJQUVatWLde2LpdL6enp8vf3l8PhMCsSAADFgtvtVnZ2tgIDA+XjU7BzZ9MKOywsTNu3b1f//v3VpEkT3XbbbXI6nTfcNj09XT/++KNZUQAAKJYaNGig8uXLF2hb0wrb4XBo1KhRkqQrV65ozZo1qlChwg239ff3l3Q1eF6lXholJCR4XlaA9zDv1mDercG8F8y+9mGSpEbrvyiS4yUkJEj6pf8KwrTC3r9/vz7++GNNmTJFcXFxatWqVZ7bXlsGdzqdCggIMCuSLTEf1mDercG8W4N5N9Zg9vuSin6ubuZlYNMK+84771ROTo569+4tp9Opt956y6yhAAAwVdmQu62OYO6S+NSpU806PAAApQp3OgMAwMCuxvW1q7G1N/+isAEAsAEKGwAAG6CwAQCwAQobAAAboLABALAB097WBQBASVHjb89aHYHCBgDASNBfh1gdgSVxAADsgDNslAq+z867uR0W7FXOtIHmjyMVahwA3vWfJ6IkSb//91zLMlDYAAAYuPTNV1ZHYEkcAAA7oLABALABChsAABugsAEAsAEuOgMAwEC5+5tbHYHCBgDASP25C6yOwJI4AAB2QGEDAGDg7Jx/6+ycf1uagSVxAAAMnHjjVUlS9ceesCwDZ9gAANgAhQ0AgA1Q2AAA2ACFDQCADZh20Vl6erpGjx6tixcv6vLlyxo6dKjCwsLMGg4AgBLNtML+9NNPVa9ePT333HM6deqUoqKiKGwAgC3d/f0+qyOYtyReuXJlJScnS5JSU1NVpUoVs4YCAMBUPk6nfJxOSzOYdobdqVMnLVmyRBEREUpNTdU//vEPs4YCAMBUmQf2S5JuubOhZRlMK+xly5YpODhYc+bM0f79+/XCCy8oJiYm330SEhLMimNb8fHxVkcotbw19/yMf8FcWIN5N+bq00OS5PN/n1mWwbTC3rlzp1q3bi1Jatiwoc6cOaMrV67Izy/vIUNCQhQQEGBWJNuJj49XaGio1TFKhgV7b3qXQs29t8Ypgfh9twbzXjC7/rsc3rSI5qow/0gy7TXs2267zXPGfPr0aQUGBuZb1gAAIG+mNWhkZKTGjBmjAQMGKDs7WxMnTjRrKAAASjzTCjswMFCzZs0y6/AAAJQq3OkMAAAb4EVlAAAM/O7d96yOQGEDAGCk4kPtrI7AkjgAAHZAYQMAYGBv2B+1N+yPlmZgSRwAAAPZZ89aHYEzbAAA7IDCBgDABihsAABsgMIGAMAGuOgMAAAD1foNtDoChQ0AgJFa4yZaHYElcQAA7IDCBgDAwNEXRunoC6MszUBhAwBgIOWzGKV8FmNpBgobAAAboLABALABChsAABugsAEAsAHehw0AgIGAur+zOgKFDQCAkYZrPrc6AkviAADYAYUNAICBlJXLlbJyuaUZWBIHAMDA0dEjJEmVu3SzLINphb148WItX/7Lv0YSEhK0c+dOs4YDAKBEM62we/furd69e0uSduzYoRUrVpg1FAAAJZ5XXsOeNWuWhgwZ4o2hAAAokUwv7N27dysoKEhBQUFmDwUAQIll+kVn0dHR6tSpU4G2TUhIMDmN/cTHx1sdodTy1tzzM/4Fc2EN5t2YKytLkrVzZXphb9++XePHjy/QtiEhIQoICDA5kX3Ex8crNDTU6hglw4K9N71LoebeW+OUQPy+W4N5L5isL7ZIkpy1ahfJ8QpT/KYW9qlTp+R0OilhAICtFVVR/xamvoZ95swZ3XrrrWYOAQCA6a6kpOhKSoqlGUw9w27SpIk++OADM4cAAMB0ex64T5LUdG+iZRm4NSkAADZAYQMAYAMUNgAANkBhAwBgAxQ2AAA2wOM1AQAwUHviZKsjUNgAABip+qe+VkdgSRwAADugsAEAMPDjo93046PdLM3AkjgAAAYy9+6xOgJn2AAA2AGFDQCADVDYAADYAIUNAIANcNEZAAAGKraLsDoChQ0AgJHfzZxtdQSWxAEAsAMKGwAAAyenv6GT09+wNAOFDQCAgTPv/1Nn3v+npRkobAAAbIDCBgDABihsAABsgMIGAMAGeB82AAAGfMoGWh2BwgYAwMjdO3ZbHcHcJfEVK1bokUceUc+ePbVp0yYzhwIAoEQzrbDT09M1Z84cLVy4UP/85z8VFxdn1lAAAJgq7dutSvt2q6UZTFsS/+qrrxQWFqaAgAAFBQVp8uTJZg0FAICpEgf1lyQ13ZtoWQbTCvvkyZPKzMzUsGHDdP78eQ0fPlwtWrTId5+EhASz4thWfHy81RFKLW/NPT/jXzAX1mDejbmysiRZO1emFXZWVpaOHz+uGTNm6NixYxo0aJA2bNggH5+8V+FDQkIUEBBgViTbiY+PV2hoqNUxSoYFe296l0LNvbfGKYH4fbcG814wu5xOSVLTIpqrwhS/aa9hV69eXffcc498fX31u9/9TuXKlVNKSopZwwEAUKKZVtgtW7bU1q1b5Xa7lZycrPT0dFWuXNms4QAAKNFMWxIPCgpS+/bt9ec//1np6ekaN25cvsvhAAAgb6beOKVPnz7q06ePmUMAAGC62z+JtjqC8ZL4tGnTdPjwYS9EAQCgeAq8N1SB91p7cZ7hGXb58uU1cuRIBQYGqlevXurYsSNXcgMA4GWGhT148GANHjxYR48e1Zo1axQVFaWGDRtq4MCBql+/vjcyAgBgqd1NG0qSmuzab1mGAl8FdubMGR09elTp6ekKDAzU6NGjtWjRIjOzAQBQLLizs+XOzrY0g+EZ9jvvvKMVK1aobt266tOnj1566SX5+voqKytLkZGRioyM9EZOQJLk++w8qyMAgCUMC/vcuXP68MMPVatWLc/njh07pjp16mjEiBGmhgMAAFfluyTucrmUmJio4OBguVwuuVwuZWRkaNiwYZKkBx980CshAQAo7fI8w165cqVmzZqlI0eOqFGjRp7P+/j4qFWrVl4JBwAArsqzsLt06aIuXbpo1qxZGj58uDczAQBQrAQNfcbqCHkX9hdffKGwsDDVqFFDS5YsyfX1Rx991NRgAAAUFzWG/s3qCHkX9oEDBxQWFqbvvvvuhl+nsAEA8J48C3vw4MGSpFdffVXZ2dny9/fXpUuXlJSUpIYNG3otIAAAVjs05ElJUr3Z71uWwfBtXTNnzlTVqlXVvXt3de/eXf7+/mrTpo2ef/55b+QDAMByFzdtsDqC8Z3ONm3apP79+2vt2rVq37691q5dq127dnkjGwAA+C/Dwi5btqwkafPmzWrXrp0kyd/f39xUAADgOgV6WteTTz6po0ePKjQ0VLGxsfL19fVGNgAA8F+GhT1t2jR98803atq0qXx8fFS+fHlNnTrVG9kAAMB/GRa2j8/VVfMvv/xSbrdbknTy5Ene1gUAKDXKNr3X6gjGhf3444/Lz89PwcHB132ewgYAlBZ3LMx9AzFvMyzs7OxsffLJJ97IAgAA8mB4lfidd96p8+fPeyMLAADF0rn5c3Vu/lxLMxieYZ88eVIPP/yw6tevLz+/XzbnrBsAUFokTXlJklRtQJRlGQr0GjZv4wIAwFqGS+ItWrRQamqq9u7dqz/84Q+qVq2aQkNDvZENAAD8l+EZ9iuvvKIzZ87o2LFjGjRokNauXauTJ0/qpZdeyne/hIQEDRkyRHXr1pUkNWjQQOPHjy+a1AAAlDKGhb1371598sknGjhwoCTp6aefVp8+fQwPnJGRoYiICI0dO/a3pwQAoJQzXBK/duMUh8MhScrJyZHL5TI8cHp6+m+MBgAArjE8w7777rs1duxYnTlzRnPmzFFcXJyaN29ueOCMjAzFx8frscceU3Z2toYOHaoWLVoUSWgAALwp5Fvrn1LpcF+732g+YmNj9d1338nhcCg0NFQPP/yw4YETExN18OBBRURE6MiRI57Xv51OZ65tL1++rISEhMJ9B7CtPyzYa3WEfH3br/FN71OY76kw4wAoGUJCQhQQEFCgbfM8wz5x4oTnz02aNFGTJk08Hx8/fly1a9fO98D169dX/fr1JUl169ZVtWrVdPr0adWpU6dIgpcG8fHxJfuK/GJe2IWa+0J8TyX6Z3wTSvzvezHFvBfMz4f+I0kqU+/3RXK8+Pj4m94nz8Lu27evHA6H3G63zpw5owoVKsjlciktLU233Xab1q5dm++BP/30U6WmpmrQoEFKTk5WcnKygoKCbjogAABWO9C5vSSp6d5EyzLkWdhffPGFJOnVV19Vjx491KhRI0nSrl27tGLFCsMDt2vXTqNGjdK6det05coVvfjiizdcDgcAAMYMLzr78ccfPWUtSU2bNtX06dMND1y+fHn985///G3pAACApAIU9pUrV/TWW2+pWbNmkqSdO3fq8uXLpgcDAAC/MHwf9ttvvy2Hw6FFixZp0aJFunLlit5++20vRAMAANcYnmFXrVpVI0aM8EYWAACQB8PCBgCgtKv79jtWR6CwAQAwUunhjlZHMH4Ne9q0aTp8+LAXogAAgLwYnmGXL19eI0eOVGBgoHr16qWOHTtyNzIAQKmyr31rSVKj9V9alsGwsAcPHqzBgwfr6NGjWrNmjaKiotSwYUMNHDjQc+tRAABKsqykJKsjGC+JX3PmzBkdPXpU6enpCgwM1OjRo7Vo0SIzswEAgP8yPMN+5513tGLFCtWtW1d9+vTRSy+9JF9fX2VlZSkyMlKRkZHeyAkAQKlmWNjnzp3Thx9+qFq1ank+d+zYMdWpU4f3ZwMA4CX5Lom7XC4lJiYqODhYLpdLLpdLGRkZGjZsmCTpwQcf9EpIAABKuzzPsFeuXKlZs2bpyJEj1z38w8fHR61atfJKOAAAioMqvftYHSHvwu7SpYu6dOmiWbNmafjw4d7MBABAsVJn0hSrI+T/POywsDDVqFFDS5YsyfX1Rx991NRgAADgF3kW9oEDBxQWFqbvvvvuhl+nsAEApcWxF1+QZO2Zdp6FPXjwYEnSq6++6rUwAAAUR+cX/5+kYlrYYWFhcjgcee64adMmM/IAAIAbyLOwFyxY4M0cKEZ8n5130/vkTBtoQhIUB/w+AMVDnoV98OBBhYWF3fCCM4nXsAEA8CbDi87i4+Nv+HUKGwAA7ynwRWfnz5+XJFWpUsULsQAAKD6cv7o9t1UM7yW+evVqTZly9aq4nJwc+fv7a/z48Wrfvr3p4QAAKA6sfA72NYaFPXv2bC1cuFB16tSRJCUmJupvf/sbhQ0AgBcZPg+7bt26nrKWpPr166t27doFOvjPP/+stm3bKiYmpvAJAQCw2IV1a3Rh3RpLM+R5hr1lyxZJUnBwsCZPnqwWLVrIx8dHW7ZsUb169Qp08H/84x+qVKlSkQQFAMAqR/5+9SmVlfYmWpYhz8KePXv2dR8fOHDA82e322144MTERCUmJqpNmzaFTwcAACTlU9jz5uV9s4T169cbHvj111/X+PHj9emnnxYuGQAA8DC86OzEiROaP3++UlJSJElZWVn69ttv873o7LPPPtN9991X4Ne6r0lISLip7UuDvN4HX9zYJefN8Nb3xNz99v3w2zDvxlxZWZKsnSvDwh49erQefPBBbdy4UZGRkYqLi9PUqVPz3WfTpk06fvy41q9fr1OnTsnpdKpGjRpq2bJlvvuFhIQoICDg5r6DEiw+Pl6hoaHeH3jB3pvepVA5CzGON3nre7LkZ3wzvPQ9Wfb7Xsox7wWzy+mUJDUtorkqTPEbFravr6/++te/6quvvlJUVJQiIyM1fPjwfMv37bff9vx51qxZqlWrlmFZAwCAvBkWdmZmpg4dOiRJOnTokGrVqqUzZ86YHgwAgOLizlXG126ZzbCwn3rqKe3atUtRUVF65JFHJMnz/wUxfPjwwqcDAKAYKFPv91ZHMC7s8PBwz5+3bdumzMxMVaxY0dRQAAAUJzlpaZIk33LlLMtgWNgHDx7UzJkzlZiYKIfDoQYNGmjYsGH6/e+t/9cGAADekPCHppKkpsXxxinXjB49Wv3799ff/vY3ud1u7dixQ6NGjdLSpUu9kQ8AAKgAhV2lShX16tXL8/Htt99eoBunAACAopPnwz9cLpdcLpdCQ0O1fv16paWlKT09XXFxcbr//vu9mREAgFIvzzPsxo0by+Fw3PC+4X5+fnrqqadMDQYAAH6RZ2Hv378/z51cLpcpYQAAwI0Zvoadlpamjz76SAkJCXI4HLr33nv15z//WWXKlPFGPgAALFfrhQlWR8j7Nexrxo4dq7S0NPXt21d9+vTR6dOnNW7cOG9kAwCgWKg2IErVBkRZmsHwDPv8+fOaMWOG5+M2bdpowIABpoayE99n834MaX5ypg0s4iQoaoX92RbXcbz5O1eY7+nbfo1NSAKUHIZn2JmZmcrIyPB8nJ6ersuXL5saCgCA4uSnvo/qp76PWprB8Aw7MjJSHTt2VJMmTSRdfWb1M888Y3owAACKi4xdO62OYFzYjz76qB544AHt2bNHkjRu3DgFBQWZHgwAAPwi38J2u90aOXKkpk+frpo1a3orEwAA+B/5FrbD4dBtt92mxYsXq1mzZnI6nZ6v1alTx/RwAADgKsMl8RUrVuT6nMPh0Oeff25KIAAAkJthYW/YsMEbOQAAKLYqtAm3OkLehZ2WlqZ3331Xhw4dUrNmzfSXv/xFfn6G/Q4AQIlTb/b7VkfI+33YEyZMkK+vr/70pz/p0KFDmjlzpjdzAQCAX8nzlPnEiRN66623JElhYWGKirL2lmwAAFjl1LtX7/hZY+jfLMuQ5xm2v7+/58++vr5yOBxeCQQAQHFz+t2ZOv2utSvNeRb2/xY0hQ0AgHXyXBLfuXOn2rRp4/k4OTlZbdq0kdvtlsPh0KZNm7wQDwAASPkUdmxsrDdzAACAfORZ2LVq1fpNB87MzNSYMWOUnJysjIwMDR06VG3btv1NxwQAoLQy7Y3VGzZsUEhIiJ588kklJSXpL3/5C4UNALAlx68uxLaKaYXduXNnz59PnTrFE74AALbVZNd+qyOYV9jX9O7dW+fOndO//vUvs4cCAKDEMr2wFy9erD179mjkyJFatmyZfHzyfCeZEhISzI5TbMTHxxfpdlazS87SzA4/IztkLImYd2PuA/skSY47G1mWwbTC/uGHH1S1alUFBwfrrrvuksvlUkpKiqpWrZrnPiEhIQoICDArkjkW7C3UbqGhoYbbxMfHF2i7IleI76lQOQs5dyicQv8uefHnZMnveyln2X9nbGbXwD9JkpruTSyS4xXmH0l5n+7+Rjt37tTcuXMlSefOnVN6eroqV65s1nAAAJRophV2ZGSkzp07p379+umpp57Siy++mO9yOAAAyJtpS+JOp1PTpk0z6/AAAJQqPODaIr7PzivYhr96/TBn2kCT0vx2Bf5+AACFwho1AAA2wBk2AAAG6n/0idURKGwAAIyU+8MfrY7AkjgAAHZAYQMAYOCH+5roh/uaWJqBJXEAAAy4MtKtjsAZNgAAdkBhAwBgAxQ2AAA2QGEDAGADXHQGAICBW598yuoIFDYAAEZqjhhldQSWxAEAsAMKGwAAA4efGaLDzwyxNANL4gAAGEiNW2t1BM6wAQCwAwobAAAboLABALABChsAABvgojMAAAzc0vguqyNQ2AAAGGmwZLnVEVgSBwDADihsAAAMJEcvVHL0QkszlNglcd9n5930PjnTBpqQBCgeCvN3orjj7zm85fjEcZKkqn/qa1kGUwv7rbfe0rZt25Sdna0nn3xSHTt2NHM4AABKLNMKe/v27dq3b5/+7//+TxcuXFC3bt0obAAACsm017Dvvfdevf3225KkChUqKDs7Wy6Xy6zhAAAo0Uw7w/bz85Of39XDL168WGFhYfLxyf/fBwkJCWbFKZD4+HhLxzdS3PMBv5U3fsf5e5Qbc2LMlZUlydq5Mv2is7i4OEVHR2vOnDmG24aEhCggIKBoBl6w96Z3CQ0N9co4hVWofIXhxe8J+LWb/h331t/zEiw+Pp45KYBdTqckqWkRzVVhit/Uwt68ebNmz56tDz74QBUqVDBzKAAATHPX1zusjmBeYV+6dEmvvfaa5s6dq8qVK5s1DAAApvMrBj1mWmGvXr1aqampGjFihOdzU6dOVXBwsFlDAgBgiqyk45IkZ63almUwrbD79OmjPn36mHV4AAC8Zl/7MElS072JlmXg1qQAANgAhQ0AgA1Q2AAA2ACFDQCADVDYAADYQIl9vCYAAEXlttenWx2BwgYAwEjlLt2sjsCSOAAAdkBhAwBgYH/Httrfsa2lGVgSBwDAwOUjh62OwBk2AAB2QGEDAGADFDYAADbAa9i/4vvsPKsj5Ku45wMAmIfCBgDAQOUej1gdgcIGAMDIbVPesDoCr2EDAGAHFDYAAAaSJk9U0uSJlmagsAEAMHBuwTydW2Dthb8UNgAANkBhAwBgAxQ2AAA2QGEDAGADvA8bAAAD/tWrWx3B3DPsH3/8Ue3atdP8+fPNHAYAAFM1/mKrGn+x1dIMphV2RkaGXn75ZbVo0cKsIQAAKDVMK2yn06n3339ft956q1lDAADgFakb45S6Mc7SDKa9hu3n5yc/P14iBwDY3+Ghf5UkNd2baFmGYtWoCQkJVkcAYKH4+PgSMYbdMCfGXFlZkqydq2JV2CEhIQoICCiagy3YWzTHAeA1oaGhN7dDIf6e3/QYJVx8fDxzUgC7nE5JUtMimqvCFD/vwwYAwAZMO8NOSEjQ1KlTlZSUJD8/P61du1azZs1SpUqVzBoSAIASy7TCDgkJ0bx51j7ZBACAkqJYvYYNAEBx1ODTVVZHoLABADByy50NrY7ARWcAABhxZWV53tplFc6wAQAw8MM9jSRx4xQA0B8W7OX+CUA+WBIHAMAGKGwAAGyAwgYAwAYobAAAbICLzgAAMBA86nmrI1DYAAAYqf7YE1ZHYEkcAAA7oLABADCQGNVPiVH9LM3AkjgAAAbStm+zOgJn2AAA2AGFDQCADVDYAADYAIUNAIANcNEZAAAGyrdsZXUEChsAACO///dcqyOwJA4AgB1Q2AAAGDj93mydfm+2pRlYEgcAwMCpGdMkSUF/HWJZBs6wAQCwAVMLe8aMGYqMjNQjjzyiH374wcyhAAAo0Uwr7K1bt+qHH37QokWL9Nprr+m1114zaygAAEo80wp727Ztatu2rSSpQYMGOnPmjDIzM80aDgCAEs20i87Onj2rhg0bej6uUqWKzp07pzp16uTa1u12S5KysrKKbPyagf5FdiwAJcfly5etjlDsMCfG3NWqSyr6ubrWfwVhWmH7+19fmG63Ww6H44bbZmdnS5J+/PHHIht/Wfc7iuxYAEqOhIQEqyMUO8xJAUx/V1LRz1V2drbKlClToG1NK+zq1asrOTnZ8/H58+dVrVq1G24bGBioBg0ayN/fP89SBwCgpHC73crOzlZgYGCB9zGtsFu3bq3p06erX79+2rNnj+rUqZPnvyJ8fHxUvnx5s6IAAFDsFPTM+hrTCjskJEQNGzZUz5495evrq1deecWsoQAAKPEc7pt5xRsAAFiCO50BAGADFDYAADbAwz+KgRkzZmjLli3KysrSpEmTdPfdd+faZtq0afr+++81b948CxKWTPnNe48ePa67EPLNN99UUFCQFTFLnPzm/dSpUxo1apR+/vlnNWrUSC+99JKFSUuWvOb99OnTeu655zzbHTt2TM8++6y6du1qVdQSJb/f9/nz52v58uXy8fFRSEiIxo4dm/87pdyw1JYtW9yPP/642+12uw8cOODu169frm1++uknd58+fdwDBgzwdrwSy2jeu3fvbkGqks9o3kePHu3eunWr2+12u1988UX38ePHvZ6xJCrIf2fcbrf7ypUr7sjISHdaWpo345VY+c37pUuX3GFhYe7s7Gy32+12Dxo0yP3dd9/lezyWxC1WkFu4Tp06VSNHjrQiXollNO/p6elWRSvRjOZ9z549at68uSRp4sSJqlWrliU5S5qC3io6JiZGbdu2van3BiNv+c27v7+//P39lZaWpitXrigzM1OVKlXK93gUtsXOnj2rKlWqeD6+dgvXa2JiYtS8eXMFBwdbEa/EMpr3CxcuaMSIEerbt6+mT59+U7cPRN7ym/eLFy8qMDBQU6ZMUf/+/TVt2jTmvYgY/b5fEx0drcjISG9GK9Hym/eAgAANGTJEERERat++ve655x7Vq1cv3+NR2BbL7xauFy5c0PLlyzVo0CALkpVsRrfOHTFihF588UV9/PHHOnDggNauXevtiCVSfvOelZWln376SVFRUZo7d6727t2rTZs2WZCy5CnIraLj4+NVs2ZNlStXzpvRSrT85j0tLU3/+te/tGbNGq1du1Y//PCD9u7dm+/xKGyL5XcL161bt+rs2bPq16+fhg0bpj179mjKlClWRS1RjG6d269fP1WqVEn+/v5q3bp1kd7nvjTLb94rV66s2rVrq1atWvLz81PLli118OBBq6KWKAW5VfSXX36p8PBwb0cr0fKb98TERNWtW1dVqlSR0+lUs2bNtGfPnnyPR2FbrHXr1vr8888lKdctXDt06KBVq1YpOjpa77zzju666y698MILVsYtMfKb9wsXLujxxx/3PJRm+/btuuMOHiZTFPKbd19fXwUHB+vYsWOSpF27dhkuEaJg8pv3a3bt2sXveRHLb96Dg4P1n//8x/OUyn379ul3v/tdvsfjbV0Wu9EtXGNiYlS+fHm1b9/e6nglltG8P/jgg4qMjJS/v7/uuusudejQwerIJYLRvD///POaMGGCMjMzdccdd3gu2MFvU5D/zpw9e1a33nqrxUlLFqN5HzRokPr16yc/Pz/de++9uv/++/M9HrcmBQDABlgSBwDABihsAABsgMIGAMAGKGwAAGyAwgYAwAZ4WxdgU2fOnFGbNm3097//XYMHD5YkDRw4UE8//bRatmzp2W769Ony8/PT8OHDNXDgQKWmpqpixYqSrt4zvXXr1vr73/8uSbpy5Yr+8Y9/KC4uTrfccouysrLUoUMHPfHEE/Lxufrv+927d+utt97SxYsX5e/vrypVquiFF15QnTp1vDsBQClDYQM29emnn6pBgwaKiYnxFHZBjBkzxlPo2dnZGjhwoDZu3KiHHnpIM2fOVFJSkqKjoxUQEKDLly9rzJgxev311zVmzBidO3dOzzzzjGbMmKGmTZtKkpYvX64nnnhCq1atkp8f/0kBzMKSOGBTMTExGjNmjDIzM7Vz585CHcPf319NmzZVYmKiMjMztWDBAk2YMEEBAQGSrj6g4MUXX9TSpUuVlpamefPmqWvXrp6ylqRu3bopOjqasgZMRmEDNvTtt9/qypUrat68uXr06KGYmJhCHSclJUVffvmlmjVrpiNHjqhmzZqe5fJrKlWqpKCgIB06dEgHDx5USEhIruP87z4Aih7/JAZsaMmSJerZs6ccDod69eqlRx55JN/7zP/6yUyvvfaaKlasqMzMTJ09e1YjR45Us2bNlJCQoJycnBvu73K55Ovrq5ycnDy3AWAuChuwmbS0NK1fv141a9bU+vXrJUk5OTlat26dAgMDlZqaet32KSkpuv322z0fX3sNOy0tTT169FDjxo0lSfXq1dOZM2d0/vz5657he+HCBSUnJ6tevXq688479f3336tTp07XjbFr1y41adIk1yMbARQdlsQBm1mxYoXuv/9+rV69WsuWLdOyZcv00ksvKSYmRuHh4YqJifE8ASgpKUmbNm3SQw89lOs45cqV05gxY/T8888rJydHgYGB6t27tyZOnKiff/5Z0tVnVL/88svq06ePbrnlFkVGRmr16tXatm2b5zirV6/W2LFjPU83A2AOHv4B2EyvXr00bNiw60o4KytLDz30kBYtWqRly5Zpw4YNCggIkNvt1rBhw9SqVStJN37b1/Dhw3X33Xdr8ODBcrlcev/997V27VrPVeIdOnTQk08+6Tl73rdvn958801duHBBTqdTNWvW1PPPP6/q1at7dyKAUobCBgDABlgSBwDABihsAABsgMIGAMAGKGwAAGyAwgYAwAYobAAAbIDCBgDABihsAABs4P8Dc27z6wEuxswAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "score = 0.782; \n",
    "#pvalue_iris = np.mean(p_auroc > score)\n",
    "pvalue = 0.0001\n",
    "ax.hist(p_auroc, bins=20, density=True)\n",
    "ax.axvline(score, ls=\"--\", color=\"r\")\n",
    "score_label = f\"Score on original\\ndata: {score:.2f}\\n(p-value: {pvalue:.3f})\"\n",
    "ax.text(0.7, 10, score_label, fontsize=12)\n",
    "ax.set_xlabel(\"AUROC\")\n",
    "_ = ax.set_ylabel(\"Probability density\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAGVCAYAAABQN+2dAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA0XUlEQVR4nO3deXxNd+L/8ffNWmINvimhxmg1NKWlM4aqkCBiX0csaXTapgs6Q4evrUqrWlq1tWbpqtaxxFq1pEo7U1K0qKU6DLFvESELieT8/vBzvzVJJLg3Hyf39Xw8+njk3nuW98kJ3v18Ts5xWJZlCQAAAMZ4mQ4AAADg6ShkAAAAhlHIAAAADKOQAQAAGEYhAwAAMIxCBgAAYBiFDAAAaM+ePYqNjVX79u0VGRmpXr16adu2baZjudXp06fVoUOHQpeLjY3Vnj17bns/MTExWr58+U2X8bntrQMAgBLBsiw999xzev3119WyZUtJ0urVqzVgwABt3LhRpUqVuqPt5+TkyNvb2xVRXSYnJ0dBQUFatWpVocvOmjXL7XkYIQMAwMNduHBBZ8+eVYMGDZzvtWvXTitWrHCWsVmzZqldu3Zq3bq1JkyYoOv3lf/ss88UFRWlyMhIvfjiizp//rwkafjw4XrjjTfUvn17rVmzRllZWRo/frwiIyPVrl07vf/++8rv3vRXrlzRmDFjFBkZqbZt22rixInKycmRJIWFhWnhwoXq0aOHmjRpogkTJuR7PCdOnNDTTz/t3NeyZcskSceOHdPjjz+uN954Q3379tWxY8dUr149SdLly5c1ZMgQhYeH65lnntHkyZM1dOhQSVJ4eLi2bdumI0eOqFmzZpozZ446dOigZs2aOQudZVmaMGGC2rRpo/DwcA0dOlTZ2dlFPgcUMgAAPFzFihX1yCOPKDY2VosWLdKxY8ckSUFBQZKknTt3as6cOVq4cKE+//xzff/99/riiy+0Y8cOffTRR5o7d67Wrl2ratWq6Z133nFuNzExUUuWLFH79u01Z84cHTp0SKtWrdLixYuVkJCgTZs25ckya9YsnTp1SqtXr9aKFSu0bds2rVy5UpLk7e2txMRELVy4UMuWLdO8efN08uTJPNt45ZVX1LhxY61du1Yffvih3njjDR09elSSlJqaqrp162rBggU3rLNkyRKdOHFC69at0/jx47Vs2bI8o3peXl5KSUmRw+HQqlWrNGrUKE2dOlWS9OWXX+pf//qXVq1apTVr1mjPnj1avXp1kc8BhQwAAOjjjz9WZGSkPvvsM0VERKh9+/Zat26dJOmrr75SWFiYypQpIz8/P82ePVuRkZHauHGjIiIiFBgYKEnq0qWL/vWvfzm32aRJE91zzz2SrhWW7t27y9fXV6VLl1bnzp21fv36PDk2btyobt26ydvbW35+foqKirphmx07dpSXl5eCgoJUuXJlnTp16ob1s7Oz9e2336pHjx6SpGrVqqlRo0basmWL8/M2bdrk2e+2bdsUGRkpHx8f3XvvvWrSpEm+36erV6+qa9eukqTQ0FDn/lu1aqX4+Hj5+fnJz89PDz/8sLMEFgXXkAEAAAUEBGjgwIEaOHCgzp07p/j4eA0ZMkTLly/X+fPnVaVKFeey16cxz58/r8qVKzvfL1++vHPK8vrr65KTkzVp0iRNnz5dkpSVlXXDFOkvl6tQoUKB2yxTpozzay8vL+d05nUpKSnKzc1VxYoVb9hGSkqKpGujbL/cxnWpqak35K1atapOnz6dZzlvb2+VLl1akuRwOJSbmyvp2i8ITJgwQfv375cknTt3TrGxsXnWLwiFDAAAD3fq1CkdP35cjRo1kiRVrlxZcXFxWrNmjQ4cOKDAwEBnoZHk/Dq/939Z0H6pSpUq6t+/vyIiIm6apVKlSkXeZn4qVqzonFq8PnKXkpKiSpUq3XS9MmXKKC0tzfk6vzJ2M1OnTpW3t7dWrFghPz8//fnPf76l9ZmyBADAw508eVIvvPCCdu3a5Xxv9+7dOnnypEJDQxUeHq4NGzbowoULunr1qgYMGKBvvvlG4eHh+uqrr5wFavHixQoLC8t3H+Hh4Vq0aJGys7NlWZZmzpypb775Js9yLVu21PLly5Wbm6vMzEx9/vnnBW4zP76+vmrWrJmWLFkiSTp69Kh27Nihpk2b3nS90NBQbdiwQbm5uTp+/Hi+2W7m4sWLCgkJkZ+fn3bt2qXvv/9e6enpRV6fETIAADzco48+qtdff13jxo1TWlqafHx8VKFCBb377rsKDg5WcHCw/vCHPyg6Olo5OTkKCwtTx44d5XA49PTTT6t3796yLEv16tXT2LFj891H3759dfz4cXXs2FE5OTmqX79+vlN6Tz75pI4ePaqoqChJ137b8/rXRTVu3DiNHj1aixcvlq+vr15//XVVrVrV+csK+YmOjlZiYqLCw8NVt25ddejQQRcvXizyPp966ikNGzZMCxcuVJMmTTRixAgNHz5cDRs2LNL6Diu/3zkFAADwMJZlyeFwSJImTpyo3NxcjRgxolj2zZQlAADweF999ZW6d++urKwspaena+PGjXr00UeLbf9MWQIAAI/XvHlzbdq0SVFRUfLy8lJERIQiIyOLbf9MWQIAABjGlCUAAIBhFDIAAHCDbdu2KTw8vNDldu7cqZ9++umO9nX8+HH169dPYWFhioqKUmJiYr7LZWdna8yYMWrZsqVatWqlzz77TNK1Ry21bdvW+V/r1q3VuHHjO8pkAteQAQCA27JkyRI1atRIISEht72N0aNHq3nz5oqLi9P333+vgQMH6ssvv3Q+DeC6v//977IsSxs2bFBycrIGDRqkTp06KTY29obbZ3z00Uc6d+7cbecxhREyAACgmTNnKiwsTF26dNG3337rfP/KlSv685//rMjISIWHh2vixImSpPnz52v58uV6++239cknn8iyLE2YMEFt2rRReHi4hg4dquzsbEnS+vXr8719xMWLF7V161b169dPktSwYUMFBwfnO0q2dOlSDRw4UA6HQ5UrV9b8+fNveMSSdO1RTgsWLNCLL77oqm9LsWGEDAAAD3fgwAF9+umnWr16tQIDAzVo0CDnZ/Pnz1dKSoq++OILXbp0SW3atFFERIR69+6t1atXq0ePHurcubMSEhL0r3/9S6tWrZJ07UHjq1evVufOndW6dWu1bt06z36PHDmiihUrOp8NKUk1atTQ4cOHb1guPT1dqampWrNmjRYtWiQvLy8999xzat++/Q3LffDBB+rZs6fKli3rwu9O8aCQAQDg4bZu3apGjRo5nxnZqVMn7du3T5LUv39/9enTR15eXipfvrweeOABHTt2TI899tgN22jVqpWeeOIJ+fn5SZIefvhhHT169Kb7vXz5snP56/z9/fM8cujSpUvKyMhQVlaWVq5cqX379qlv374KCQlR7dq1JUkZGRlasWKF1qxZc/vfCIMoZAAAeLjU1FSVL1/e+fqXXx88eFATJ05UUlKSHA6HTp06pW7duuXZxunTpzVhwgTt379fknTu3Ll8H430S6VKlbrhgd6SlJaWpoCAgBveK1eunK5evaonn3xSDodD9erV0+9+9ztt2bLFWcjWrVun0NBQW46OSVxDBgCAxytXrpwuXbrkfH3+/Hnn1+PGjdP999+vL774QmvWrFHdunXz3cbUqVPl7e3tHKVq0aJFofutWbOm0tPTdeHCBed7hw8f1q9//esblitdurQqVap0w7MlHQ6HvL29na+vP+zcrihkAAB4uIYNG2rbtm06f/68cnJytGLFCudnFy9e1EMPPSQvLy999dVXSkpKck4p+vj4OIvcxYsXFRISIj8/P+3atUvff/99nqnH/1amTBk1a9ZMc+fOlSQlJiYqJSUl39tWdO7cWR9++KEkKSkpSVu3br1huR9//FH33XffnX0jDOJO/QAAQFOnTtXSpUtVrlw5dezYUXPnztWmTZu0du1avf766woICFBUVJSqVKmi6dOn669//av27t2rd955R9HR0YqIiNCwYcPk5eWlJk2aqHnz5ho+fLjefPNNORwObdiwQW+++Wae/Z4+fVpDhw5VUlKSypYtq9dff935DMm2bdtqzpw5qly5stLS0jRq1Cj9+OOP8vPz0wsvvKDOnTs7t/Poo49q8eLFzilMu6GQAQAAGMaUJQAAgGEUMgAAAMMoZAAAAIZRyAAAAAyjkAEAAEnS3Llz9b//+79u38+DDz6oU6dOuWXb3377rdq2basWLVooJiamwP0UtJxlWZo0aZKaN2+usLAwTZ482bnO8ePH1a9fP4WFhSkqKsr5zM2XX35ZCxcuvLPgFgAA8HhHjhyxnnjiCevSpUtu31edOnWskydPuny7aWlpVuPGja1t27ZZlmVZf/vb36xnnnnmlpZbvny51bVrVysjI8PKzMy0unTpYn3++eeWZVlW//79rb/97W+WZVnW9u3brSZNmlgZGRlWamqq1axZM+vEiRO3nZ0RMgAAoA8++EDdu3dXmTJltHnzZnXo0EETJ05U27Zt1a5dO+3atSvPOhs3blTHjh1veK9Tp076+uuvlZKSori4OEVGRioiIkKffPJJnvXj4+PVv3//fF9nZWVp/PjxioyMVLt27fT+++/L+v936po8ebLmz5+fZ3tbtmxRzZo11ahRI0lS3759tXnz5jyPZ7rZcuvWrdPvf/97lSpVSvfcc4969uypNWvW6OLFi9q6dav69esn6drNdIODg5WYmKhy5cqpa9eu+vjjj4v43c6LQgYAgIezLEsJCQlq3bq1JMnLy0v/+c9/9Nhjj2nNmjXq16+fxo0bl2e9pk2b6uTJk86HiB89elSnT59WkyZNNHPmTP3P//yP1q5dq08//VSTJ0/WyZMni5xpzpw5OnTokFatWqXFixcrISFBmzZtknRtirB379551klKSlL16tWdrwMCAlShQgUdOXKkyMslJSWpRo0azs9q1Kihw4cP68iRI6pYsaJKly6d5zPp2sPV161bV+Tj+28UMgAAPNyxY8ecjz66rnTp0oqIiJB07Y75e/bsUXZ29g3r+fn5qWXLltqwYYMkKSEhQa1atZKvr69GjhypMWPGSLpWXKpUqaJjx44VOdOXX36p7t27y9fXV6VLl1bnzp21fv36m66TmZkpPz+/PBkzMjKKvNx/f+bv76+MjAxdvnw5zzr+/v7Ox0M99NBDOnv2rE6fPl3kY/wln9taCwAAlBgpKSmqWLGivLz+b5ymXLlyzq/Lli0ry7J06dIlDR482Fk61qxZo8jISM2ePVuxsbFKSEjQiy++KEn64YcfNGXKFJ05c0YOh0Nnz55Vbm5ukTMlJydr0qRJmj59uqRrU5gNGjS46TqlS5fO8/zM9PT0G0a1CluuVKlSN0xxpqWl5fv+9c8CAgIkSd7e3ipfvrySk5MVFBRU5OO8jkIGAADySE1NlWVZcjgcunTpkhwOh8qVK6dZs2bdsFzz5s01atQoHT58WIcPH3Y+8Hvo0KF66qmnnNdcNW/ePM8+vLy8nNeFSbqhJFWpUkX9+/d3jtIVRa1atbR06VLn6/Pnzys9PT3PQ8dvtlytWrWUlJTk/CwpKUm1a9dWzZo1lZ6ergsXLqhChQqSpMOHD6tnz543bNu6zSdSMmUJAICHq1ixoi5cuHBDmcjIyFBCQoKkayNhjzzyiHx88o7j+Pn56fHHH9fbb7+tiIgI5zIXL15UaGioJGnhwoXKyMjIMyoVFBSko0ePKjs7W1evXnXuT5LCw8O1aNEiZWdny7IszZw5U998881Nj+N3v/udkpOTtXXrVknSvHnzFBYWpjJlyhR5ufbt22vJkiXKyMhQRkaG4uPj1b59e5UpU0bNmjXT3LlzJUmJiYlKSUlxFtCcnBylpqaqUqVKhXy388cIGQAAHq569eoqU6aM9u/f77yOrHr16vruu+/0zjvvyNfXV5MmTSpw/bZt22rQoEH69NNPne+99NJLev755xUYGKi+ffuqd+/eGjNmjB544AHnMo0bN1ZoaKi6dOmi6tWrq3Hjxtq8ebOka7/5ePz4cXXs2FE5OTmqX7++YmNjJV37Lctq1arlubD/nnvu0bRp0zRmzBilp6erdu3amjhxoiRp165dmjZtmj766KObLhcZGal9+/YpKipKktSzZ0+1atVKkjRu3DgNHTpUCxcuVNmyZTVjxgz5+/tLkvbs2aNKlSrp3nvvva1z4LBud2wNAACUGK+88oqCgoI0cOBAJSYmavTo0YVeRI//M2XKFF26dMn5iwy3iilLAACguLg4LV68OM+F6yjcpUuXFB8fr6effvq2t0EhAwAAqlGjhp599lmNHz/edBTbGTt2rAYMGKDg4ODb3gZTlgAAAIYxQgYAAGAYhQwAAMAwChkAAIBhFDIAAADDKGQAAACGUcgAAAAMo5ABAAAYRiEDAAAwjEIGAABgGIUMAADAMAoZAACAYRQyAAAAwyhkAAAAhlHIAAAADKOQAQAAGEYhAwAAMIxCBgAAYBiFDAAAwDAKGQAAgGE+pgNIUm5urtLT0+Xr6yuHw2E6DgAAbmVZlrKzsxUQECAvL8ZGcJcUsvT0dP3888+mYwAAUKzq1KmjsmXLmo6Bu8BdUch8fX0lXfvB9PPzM5pl9+7dCg0NNZrBJI6f4/fU4/fkY5c4flce/77WYZKkuus3FbhMVlaWfv75Z+e/f8BdUciuT1P6+fnJ39/fcBrdFRlM4vg5fk/lyccucfyuOv46Mz8o8va4TAfX3RWFDACAkqJ06MOmI8CGuJIQAADAMAoZAAAutLNebe2sV9t0DNgMhQwAAMAwChkAAIBhFDIAAADDKGQAAACGUcgAAAAM4z5kAAC40L1/fNl0BNgQhQwAABcKeu5F0xFgQ0xZAgAAGMYIGQCP4/3y7Pw/mLf3traXMznmDtKgpPnPM7GSpF9/OMtwEtgJhQwAABe69O0/TUeADTFlCQAAYBiFDAAAwDAKGQAAgGEUMgAAAMO4qB8AABcq85vGpiPAhihkAAC4UO1Z80xHgA0xZQkAAGAYhQwAABc6+8mHOvvJh6ZjwGaYsgQAwIVOvP2mJKnKU88YTgI7YYQMAADAMAoZAACAYRQyAAAAwyhkAAAAhlHIAAAADOO3LAEAcKGHd+wzHQE2RCEDAMCFvPz8TEeADTFlCQCAC2Xu/0mZ+38yHQM2wwgZAAAu9HPX9pKkBnsPGk4CO2GEDAAAwDAKGQAAgGFum7JMT0/XsGHDdPHiRV25ckUDBgxQWFiYu3YHAABgW24rZEuXLlWtWrX05z//WadOnVJsbCyFDAAAIB9um7KsWLGikpOTJUmpqakKDAx0164AAABszW0jZO3atdPixYsVGRmp1NRU/eUvf3HXrgAAuGv86v2/mY4AG3JYlmW5Y8PLli3T1q1b9cYbb+inn37SyJEjFR8fn++yV65c0e7du90RAwDy+O28vS7d3nd96rl0e/AcoaGh8vf3Nx0DdwG3jZD98MMPat68uSQpJCREZ86c0dWrV+XjU/Au74YfzO3bt6tRo0ZGM5jE8XP8HnH8Li5kJeF75jHnvgDFffwMROC/ue0asvvuu8/5w3b69GkFBATctIwBAFAS7A37nfaG/c50DNiM2xpSdHS0hg8frn79+ik7O1tjx451164AALhrZJ89azoCbMhthSwgIEAzZsxw1+YBAABKDO7UDwAAYBiFDAAAwDAKGQAAgGH82iMAAC5UuU+M6QiwIQoZAAAuFDx6rOkIsCGmLAEAAAyjkAEA4EJHRg7VkZFDTceAzVDIAABwoZRl8UpZlv+zm4GCUMgAAAAMo5ABAAAYRiEDAAAwjEIGAABgGPchAwDAhfxr/sp0BNgQhQwAABcK+eJL0xFgQ0xZAgAAGEYhAwDAhVJWrVDKqhWmY8BmmLIEAMCFjgwbLEmq2KGT4SSwE0bIAAAADKOQAQAAGEYhAwAAMIxCBgAAYBiFDAAAwDB+yxIAABequ36T6QiwIQoZAAAu5Bdc3XQE2BBTlgAAuNDVlBRdTUkxHQM2wwgZAAAutOfxxyRJDfYeNJwEdsIIGQAAgGEUMgAAAMMoZAAAAIZRyAAAAAzjon7AZrxfnu3S7eVMjnHp9gAAt45CBgCAC1UfO950BNgQhQwAABeq9PvepiPAhriGDAAAwDAKGQAALvRzj076uUcn0zFgM0xZAgDgQpl795iOABtihAwAAMAwChkAAIBhFDIAAADDKGQAAACGcVE/AAAuVL5VpOkIsCEKGQAALvSr6TNNR4ANMWUJAABgGIUMAAAXOjnlbZ2c8rbpGLAZChkAAC505oO/6swHfzUdAzZDIQMAADCMQgYAAGAYhQwAAMAwChkAAIBh3IcMAAAX8iodYDoCbIhCBgCACz28bZfpCLAhpiwBAAAMo5ABAOBCad9tUdp3W0zHgM0wZQkAgAsd7N9XktRg70HDSWAnjJABAAAYRiEDAAAwzK2FbOXKlerWrZu6du2qjRs3unNXAAAAtuW2Qpaenq5PPvlE8+fP11//+lclJCS4a1cAAAC25rZC9s9//lNhYWHy9/dXUFCQxo8f765dAQAA2Jrbfsvy5MmTyszM1MCBA3X+/HkNGjRITZo0cdfuAAC4K9w/d6HpCLAhh2VZljs2/Pe//127du3StGnTdPToUfXv318bNmyQl1feQbkrV65o9+7d7ogBlDi/nbfXpdv7rk89l27PDvge4m4RGhoqf39/0zFwFyh0hGzy5Mnq3r27fvWrX93ShqtUqaJHHnlE3t7e+tWvfqUyZcooJSVFlSpVKnCdu+EHc/v27WrUqJHRDCZx/DY4fheXiV8ery2O3xXc+D20K4859wUo7uNnIAL/rdBryMqWLashQ4YoJiZGy5Yt05UrV4q04aZNm2rLli2yLEvJyclKT09XxYoV7zgwAAB3s10NQrSrQYjpGLCZQkfI4uLiFBcXpyNHjuiLL75QbGysQkJCFBMTo9q1axe4XlBQkFq3bq0nn3xS6enpGj16dL7TlQAAlCRWdrbpCLChIl/Uf+bMGR05ckTp6ekKCAjQsGHD1LNnT0VHRxe4Tq9evdSrVy+XBAUAACipCi1k7733nlauXKmaNWuqV69eeu211+Tt7a2srCxFR0fftJABAACgcIUWsnPnzunjjz9WcHCw872jR4+qRo0aGjx4sFvDAQAAeIKbXtSVm5urgwcPqlq1asrNzVVubq4yMjI0cOBASdITTzxRLCEBAABKsgJHyFatWqUZM2YoKSlJdevWdb7v5eWlZs2aFUs4AADsJmjAS6YjwIYKLGQdOnRQhw4dNGPGDA0aNKg4MwEAYFv3Dvij6QiwoQIL2aZNmxQWFqZ7771XixcvzvN5jx493BoMAADAUxRYyPbv36+wsDB9//33+X5OIUNJ9Nt5e11+F3cAnuXQi89KkmrN/MBwEthJgYUsLi5OkvTmm28qOztbvr6+unTpko4fP66QEO5ADABAfi5u3GA6Amyo0NteTJ8+XZUqVVLnzp3VuXNn+fr6qkWLFhoxYkRx5AMAACjxCn2W0caNG9W3b1+tXbtWrVu31tq1a7Vz587iyAYAAOARCi1kpUuXliR98803atWqlSTJ19fXvakAAAA8SKFTlmXLltWzzz6rI0eOqFGjRlqzZo28vb2LIxsAAIBHKLSQTZ48Wd9++60aNGggLy8vlS1bVhMnTiyObAAA2E7pBo+ajgAbKrSQeXldm9X8+uuvZVmWJOnkyZPc9gIAgHw8MD/vvTuBwhRayJ5++mn5+PioWrVqN7xPIQMAAHCNQgtZdna25s6dWxxZAACwvXNzZkmSKveLNZwEdlJoIXvwwQd1/vx5BQYGFkceAABs7fiE1yRRyHBrCi1kJ0+eVJs2bVS7dm35+Pzf4oyaAQAAuEaRriHjNhcAAADuU+iNYZs0aaLU1FTt3btXv/3tb1W5cmU1atSoOLIBAAB4hEIL2RtvvKFVq1ZpxYoVkqS1a9dq3Lhxbg8GAADgKQotZHv37tW0adMUEBAgSXrhhRe0f/9+twcDAADwFEW+MazD4ZAk5eTkKDc3172pAACwqdDvdpqOABsqtJA9/PDDGjVqlM6cOaNPPvlECQkJaty4cXFkAwDAdrzLlDEdATZUaCEbNmyY1qxZo4CAAJ06dUqxsbFq06ZNcWQDAMB2Lh/6jyTpnlq/NpwEdlJgITtx4oTz6/r166t+/frO18eOHVP16tXdmwwAABva3761JKnB3oOGk8BOCixkvXv3lsPhkGVZOnPmjMqVK6fc3FylpaXpvvvu09q1a4szJ5Av75dnm44AuPznMGdyjEu3B+DuV2Ah27RpkyTpzTffVJcuXVS3bl1J0s6dO7Vy5criSQcAAOABCr3txc8//+wsY5LUoEEDHThwwK2hAAAAPEmhF/VfvXpV7777rho2bChJ+uGHH3TlyhW3BwMAAPAUhY6QTZ06VQ6HQwsWLNCCBQt09epVTZ06tRiiAQAAeIZCR8gqVaqkwYMHF0cWAABsr+bU90xHgA0VWsgAAEDRVWgTZToCbKjQKUsAAAC4V6GFbPLkyTp8+HAxRAEAwP72tW6ufa2bm44Bmyl0yrJs2bIaMmSIAgIC1L17d0VFRcnf3784sgEAYDtZx4+bjgAbKrSQxcXFKS4uTkeOHNEXX3yh2NhYhYSEKCYmRrVr1y6OjAAAACVaka8hO3PmjI4cOaL09HQFBARo2LBhWrBggTuzAQAAeIRCR8jee+89rVy5UjVr1lSvXr302muvydvbW1lZWYqOjlZ0dHRx5AQAACixCi1k586d08cff6zg4GDne0ePHlWNGjW4PxkAAIAL3HTKMjc3VwcPHlS1atWUm5ur3NxcZWRkaODAgZKkJ554olhCAgBgF4E9eymwZy/TMWAzBY6QrVq1SjNmzFBSUtINDxf38vJSs2bNiiUcAAB2U2PcBNMRYEMFFrIOHTqoQ4cOmjFjhgYNGlScmQAAADxKgYVs06ZNCgsL07333qvFixfn+bxHjx5uDQYAgB0dfXWkJEbKcGsKLGT79+9XWFiYvv/++3w/p5ABAJDX+UX/kEQhw60psJDFxcVJkt58881iCwMAAOCJCixkYWFhcjgcBa64ceNGd+QBUMy8X5594xvz9t7R9nImx9zR+gDgiQosZPPmzSvOHAAAAB6rwEJ24MABhYWF5XtBv8Q1ZAAAAK5S6EX927dvz/dzChkAAHn5/eLJNkBRFfmi/vPnz0uSAgMDiyEWAAD2VHf916YjwIYKfZbl6tWrNWHCtV/dzcnJka+vr1555RW1bt3a7eEAAAA8QaGFbObMmZo/f75q1KghSTp48KD++Mc/UsgAAMjHhXVfSJIqtIkynAR2Umghq1mzprOMSVLt2rVVvXp1t4YCAMCukv40UJJUYe9Bw0lgJwUWss2bN0uSqlWrpvHjx6tJkyby8vLS5s2bVatWrWILCAAAUNIVWMhmzpx5w+v9+/c7v7Ysy32JAAAAPEyBhWz27NkFfaT169cXaeOXL19W+/btNWDAAHXr1u3W0wEAAHiAQq8hO3HihObMmaOUlBRJUlZWlr777rsiXdT/l7/8RRUqVLjjkAAAACWZV2ELDBs2TOXLl9eOHTsUEhKiM2fOaOLEiYVu+ODBgzp48KBatGjhipwAAAAlVqEjZN7e3nruuef0z3/+U7GxsYqOjtagQYPUtGnTm643adIkvfLKK1q6dGmRw+zevbvIy7pTQU8n8BSefvy4M/z83DlT30NPP3euOn5r4hSXbg+eodBClpmZqUOHDkmSDh06pODgYJ05c+am6yxbtkyPPfbYLd8eIzQ0VP7+/re0jqtt375djRo1MprBJNsd/7y9phPgv9ji5+cu/7kx8T203Z99F3Pp8RdhO1euXLlrBiFwdyi0kD3//PPauXOnYmNjnRfmF3aB/saNG3Xs2DGtX79ep06dkp+fn+69995CR9UAALC7nLQ0SZJ3mTKGk8BOCi1k4eHhzq8TExOVmZmp8uXL33SdqVOnOr+eMWOGgoODKWMAAI+w+7cNJEkNuDEsbkGhhezAgQOaPn26Dh48KIfDoTp16mjgwIH69a9/XRz5AAAASrxCC9mwYcPUt29f/fGPf5RlWdq2bZuGDh2qJUuWFGkHgwYNuuOQAAAAJVmhhSwwMFDdu3d3vr7//vuLfGNYAAAAFK7A+5Dl5uYqNzdXjRo10vr165WWlqb09HQlJCToN7/5TXFmBAAAKNEKHCGrV6+eHA5Hvs+t9PHx0fPPP+/WYACAu5f3ywU/Xu925EyOcen2ALspsJD99NNPBa6Um5vrljAAANhd8MgxpiPAhgq9hiwtLU2ffvqpdu/eLYfDoUcffVRPPvmk7rnnnuLIBwCArVTuF2s6Amyo0GdZjho1Smlpaerdu7d69eql06dPa/To0cWRDQAAwCMUOkJ2/vx5TZs2zfm6RYsW6tevn1tDAQBgV//u3UOS9MD8xYaTwE6K9CzLjIwMlS5dWpKUnp6uK1euuD0YAAB2lLHzB9MRYEOFFrLo6GhFRUWpfv36kqTdu3frpZdecnswAAAAT1FoIevRo4cef/xx7dmzR5I0evRoBQUFuT0YAACAp7hpIbMsS0OGDNGUKVNUtWrV4soEAADgUW5ayBwOh+677z4tWrRIDRs2lJ+fn/OzGjVquD0cAACAJyh0ynLlypV53nM4HPryyy/dEggAADsr1yLcdATYUKGFbMOGDcWRAwCAEqHWzA9MR4ANFVjI0tLS9P777+vQoUNq2LCh/vCHP8jHp9D+BgAAgFtU4J36x4wZI29vb/3+97/XoUOHNH369OLMBQCALZ16f5pOvT+t8AWBXyhwyOvEiRN69913JUlhYWGKjeXZXAAAFOb0+9cGMO4d8EfDSWAnBY6Q+fr6Or/29vaWw+EolkAAAACepsBC9t8FjEIGAADgHgVOWf7www9q0aKF83VycrJatGghy7LkcDi0cePGYogHAABQ8hVYyNasWVOcOQAAADxWgYUsODi4OHMAAFAiOH5xDTZQVNxYDAAAF6q/8yfTEWBDBV7UDwAAgOJBIQMAwIXSf9iu9B+2m44Bm2HKEgAAFzrQ9/eSpAZ7DxpOAjthhAwAAMAwChkAAIBhFDIAAADDKGQAAACGUcgAAAAM47csAQBwodqfzjUdATZEIQMAwIXK/PZ3piPAhpiyBAAAMIxCBgCAC/34WH39+Fh90zFgM0xZAgDgQrkZ6aYjwIYYIQMAADCMQgYAAGAYhQwAAMAwChkAAIBhXNQPAIAL/c+zz5uOABuikAEA4EJVBw81HQE2RCErQbxfnu2aDc3bK0nKmRzjmu0Bd8hlP9s24Y7j5c8zcHfjGjIAAFzo8Esv6vBLL5qOAZthhAwAABdKTVhrOgJsiBEyAAAAwyhkAAAAhlHIAAAADKOQAQAAGMZF/QAAuFCpeg+ZjgAbopABAOBCdRavMB0BNsSUJQAAgGEUMgAAXCh54XwlL5xvOgZshilLAABc6NjY0ZKkSr/vbTgJ7IQRMgAAAMMoZAAAAIa5dcry3XffVWJiorKzs/Xss88qKirKnbsDAACwJbcVsq1bt2rfvn36xz/+oQsXLqhTp04UMgAAgHy4bcry0Ucf1dSpUyVJ5cqVU3Z2tnJzc921OwAAANty2wiZj4+PfHyubX7RokUKCwuTl9fN+9/u3bvdFeeWbN++3XSEuwLfB9wOfm7uTkU5LybP3d3wc+OqDNbMD126PXgGt9/2IiEhQQsXLtQnn3xS6LKhoaHy9/d3d6Sb2r59uxo1amQ0w22bt9elm7PF98HFx4w755afG87zHSvsvNzy330l7O+b4v67/8qVK3fNIATuDm4tZN98841mzpypjz76SOXKlXPnrgAAuCtkHT8mSfILrm44CezEbYXs0qVLeuuttzRr1ixVrFjRXbsBAOCusq91mCSpwd6DhpPATtxWyFavXq3U1FQNHjzY+d7EiRNVrVo1d+0SNuD98mzTEeBmnOO7U5HOC1PDgDFuK2S9evVSr1693LV5AACAEoM79QMAABhGIQMAADCMQgYAAGCY2+9DBgCAJ7lv0hTTEWBDFDIAAFyoYodOpiPAhpiyBAAAMIxCBgCAC/0UFaGfoiJMx4DNMGUJAIALXUk6bDoCbIgRMgAAAMMoZAAAAIZRyAAAAAyjkAEAABjGRf0AALhQxS7dTEeADVHIAABwofsmvG06AmyIKUsAAADDKGQAALjQ8fFjdXz8WNMxYDMUMgAAXOjcvNk6N2+26RiwGQoZAACAYVzUb5D3y3f3/0Hd7fkAACgpGCEDAAAwjEIGAABgGFOWAAC4kG+VKqYjwIYoZAAAuFC9TVtMR4ANMWUJAABgGIUMAAAXSv0qQalfJZiOAZthyhIAABc6POA5SVKDvQcNJ4GdMEIGAABgGIUMAADAMAoZAACAYRQyAAAAwyhkAAAAhvFblgAAuFCdpZ+bjgAbopABAOBCpR4MMR0BNsSUJQAALpSblaXcrCzTMWAzjJABAOBCPz5SVxI3hsWtYYQMAADAMAoZAACAYRQyAAAAw0rsNWTeL8++/ZXn7XVdEACAEbf870Ahf/fnTI65gzTAzTFCBgAAYFiJHSEDAMCEakNHmI4AG6KQAQDgQlWeesZ0BNgQU5YAAACGUcgAAHChg7F9dDC2j+kYsBmmLAEAcKG0rYmmI8CGGCEDAAAwjEIGAABgGIUMAADAMAoZAACAYVzUDwCAC5Vt2sx0BNgQhQwAABf69YezTEeADTFlCQAAYBiFDAAAFzr9t5k6/beZpmPAZpiyBADAhU5NmyxJCnruRcNJYCeMkAEAABjm1kI2bdo0RUdHq1u3bvrxxx/duSsAAADbclsh27Jli3788UctWLBAb731lt566y137QoAAMDW3FbIEhMTFRERIUmqU6eOzpw5o8zMTHftDgAAwLbcdlH/2bNnFRIS4nwdGBioc+fOqUaNGnmWtSxLkpSVleWy/VcN8HXZtgAA7nXlyhWXb9PV/w4UNaNVuUqhy1//9+76v3+A2wqZr++NfxAsy5LD4ch32ezsbEnSzz//7LL9L+/8gMu2BQBwr927d7t8m67+d6DIGae8X+Tls7Ozdc8999xJLJQQbitkVapUUXJysvP1+fPnVbly5XyXDQgIUJ06deTr61tgaQMAoKSwLEvZ2dkKCAgwHQV3CbcVsubNm2vKlCnq06eP9uzZoxo1ahT4fwFeXl4qW7asu6IAAHDXYWQMv+S2QhYaGqqQkBB17dpV3t7eeuONN9y1KwAAAFtzWFxRCAAAYBR36gcAADCMQgYAAGCYxz1cfNq0adq8ebOysrI0btw4Pfzww87P/vGPf2jJkiVyOBx68MEHNW7cOO3Zs0cvvviiatasKenaTW5feeUVU/Hv2M2O/7rJkydrx44dmj17dpHXsYNbPfbdu3d7zLnv0qXLDb9Y88477ygoKKjEnHvp1o//7NmzHnP+T506paFDh+ry5cuqW7euXnvttULXsZtbPf6S9ucfNmB5kM2bN1tPP/20ZVmWtX//fqtPnz7OzzIyMqyYmBgrKyvLsizLiomJsbZv324lJiZa48ePN5LX1W52/Nf9+9//tnr16mX169evyOvYwe0cuyed+86dO9/yOnZyO8fvSed/2LBh1pYtWyzLsqxXX33VOnbsmEed//yOvySdf9iDR01Z3uxxTqVKldJnn30mX19fZWZmKj09XVWqVFF6errJyC5VlMdZTZw4UUOGDLmldezgdo7dk859fsdaUs69dHvH70nnf8+ePWrcuLEkaezYsQoODvao85/f8Zek8w978KhCdvbsWQUGBjpfX3+c0y/9/e9/V0REhNq1a6caNWooIyND27dv11NPPaV+/fpp8+bNxR3bZQo7/vj4eDVu3FjVqlUr8jp2cTvH7knn/sKFCxo8eLB69+6tKVOmyLKsEnPupds7fk85/xcvXlRAQIAmTJigvn37avLkyR51/gs6/pJ0/mEPHnUNWVEe5xQXF6eYmBjFxcWpfv36CgkJ0XPPPafIyEglJSWpf//+Wrt2rfz8/Iozukvc7PgvXLigFStW6MMPP9SpU6eKtI6d3M6xe8q5l6TBgwerXbt2CggI0KBBg7R27doSc+6l2zt+Tzn/WVlZ+ve//613331XQUFBeu6557Rx40aPOf8FHX9JOv+wB48qZDd7nNOFCxe0f/9+NW7cWKVKlVJYWJh27NihZ599VrVr15Yk1axZU5UrV9bp06fzfUj63e5mx79lyxadPXtWffr0UVZWlo4cOaIJEybc0iOw7ma3c+wjR470iHMvSX369HF+3bx5c/38888l5txLt3f8bdu29YjzX7FiRVWvXl3BwcGSpKZNm+rAgQMec/4LOv6WLVuWmPMPe/CoKcvmzZvryy+/lKQ8j3PKzc3VqFGjlJGRIUnatWuXatWqpaVLl+rTTz+VJCUnJys5OVlBQUFG8t+pmx1/27Zt9fnnn2vhwoV677339NBDD2nkyJE3XcdObufYPeXcX7hwQU8//bSys7MlSVu3btUDDzxQYs69dHvH7ynn39vbW9WqVdPRo0clSTt37lStWrU85vwXdPwl6fzDHjxqhCy/xznFx8erbNmyat26tQYOHKgnn3xSPj4+evDBBxUREaG0tDQNHTpU69at09WrV/Xqq6/adsi6sOMv6jp2dDvH3qpVK48590888YSio6Pl6+urhx56SG3btpXD4SgR5166veP3pD/7I0aM0JgxY5SZmakHHnhAERERHnX+8zv+knT+YQ88OgkAAMAwj5qyBAAAuBtRyAAAAAyjkAEAABhGIQMAADCMQgYAAGCYR932AihJzpw5oxYtWuhPf/qT4uLiJEkxMTF64YUX1LRpU+dyU6ZMkY+PjwYNGqSYmBilpqaqfPnykq49r7F58+b605/+JEm6evWq/vKXvyghIUGlSpVSVlaW2rZtq2eeeUZeXtf+/23Xrl169913dfHiRfn6+iowMFAjR47khpkAcAcoZIBNLV26VHXq1FF8fLyzkBXF8OHDnYUtOztbMTEx+uqrr9SyZUtNnz5dx48f18KFC+Xv768rV65o+PDhmjRpkoYPH65z587ppZde0rRp09SgQQNJ0ooVK/TMM8/o888/l48Pf6UAwO1gyhKwqfj4eA0fPlyZmZn64Ycfbmsbvr6+atCggQ4ePKjMzEzNmzdPY8aMkb+/vyTJ399fr776qpYsWaK0tDTNnj1bHTt2dJYxSerUqZMWLlxIGQOAO0AhA2zou+++09WrV9W4cWN16dJF8fHxt7WdlJQUff3112rYsKGSkpJUtWpV53TmdRUqVFBQUJAOHTqkAwcOKDQ0NM92/nsdAMCt4X9pARtavHixunbtKofDoe7du6tbt24aOXJkgcs7HA7n12+99ZbKly+vzMxMnT17VkOGDFHDhg21e/du5eTk5Lt+bm6uvL29lZOTU+AyAIDbRyEDbCYtLU3r169X1apVtX79eklSTk6O1q1bp4CAAKWmpt6wfEpKiu6//37n6+vXkKWlpalLly6qV6+eJKlWrVo6c+aMzp8/r8DAQOfyFy5cUHJysmrVqqUHH3xQO3bsULt27W7Yx86dO1W/fv0bih8AoOiYsgRsZuXKlfrNb36j1atXa/ny5Vq+fLlee+01xcfHKzw8XPHx8crKypIkHT9+XBs3blTLli3zbKdMmTIaPny4RowYoZycHAUEBKhnz54aO3asLl++LEnKysrS66+/rl69eqlUqVKKjo7W6tWrlZiY6NzO6tWrNWrUKGVnZxfPNwAASiAeLg7YTPfu3TVw4MAbSlZWVpZatmypBQsWaPny5dqwYYP8/f1lWZYGDhyoZs2aScr/thiDBg3Sww8/rLi4OOXm5uqDDz7Q2rVrnb9l2bZtWz377LPO0a99+/bpnXfe0YULF+Tn56eqVatqxIgRqlKlSvF+IwCgBKGQAQAAGMaUJQAAgGEUMgAAAMMoZAAAAIZRyAAAAAyjkAEAABhGIQMAADCMQgYAAGAYhQwAAMCw/wd+gDJngmsPVQAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 576x396 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "score = 0.666; \n",
    "#pvalue_iris = np.mean(p_auroc > score)\n",
    "pvalue = 0.0001\n",
    "ax.hist(p_auroc, bins=20, density=True)\n",
    "ax.axvline(score, ls=\"--\", color=\"r\")\n",
    "score_label = f\"Score on original\\ndata: {score:.2f}\\n(p-value: {pvalue:.3f})\"\n",
    "ax.text(0.7, 10, score_label, fontsize=12)\n",
    "ax.set_xlabel(\"AUROC\")\n",
    "_ = ax.set_ylabel(\"Probability density\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import *\n",
    "def print_metrics_with_threshold_tuning(name, X_test,y_test, cols, best_model):\n",
    "    y_probs = best_model.predict_proba(X_test[cols])[:,1]\n",
    "    thresholds = arange(0, 1, 0.001).tolist()\n",
    "    scores = [roc_auc_score(y_test, convert_to_labels(y_probs, t)) for t in thresholds]\n",
    "    ix = argmax(scores)\n",
    "    print(\"Threshold: \", thresholds[ix])\n",
    "    y_test_predictions = np.where(best_model.predict_proba(X_test[cols])[:,1] > thresholds[ix], 1, 0)\n",
    "    y_test_predictions=best_model.predict(X_test[cols])\n",
    "    fp_rate, tp_rate, thresh = roc_curve(y_test, y_probs)\n",
    "    y_test_predictions = best_model.predict(X_test[cols])\n",
    "    auc = roc_auc_score(y_test, y_probs)\n",
    "    sen= sensitivity_score(y_test, y_test_predictions, pos_label=1)\n",
    "    spe=specificity_score(y_test,y_test_predictions,pos_label=1)\n",
    "    acc=balanced_accuracy_score(y_test,y_test_predictions)\n",
    "    mcc=matthews_corrcoef(y_test, y_test_predictions)\n",
    "    auprc = average_precision_score(y_test, y_test_predictions)\n",
    "    f1 = f1_score(y_test, y_test_predictions, average=\"weighted\")\n",
    "    #print(classification_report(y_test, y_test_predictions))\n",
    "    print(\"Name: \",name,\"Sensitivity: \",np.round(sen,3),\", Specificity: \",np.round(spe,3),\"AUPRC: \",np.round(auprc,3),\", AUROC: \",np.round(auc,3),\", MCC: \",np.round(mcc,3),\" , F1-weighted: \", np.round(f1,3) )\n",
    "    return(y_test_predictions, y_probs, np.round(sen,3), np.round(spe,3),np.round(auprc,3), np.round(acc,3), np.round(auc,3), np.round(mcc,3), np.round(f1,3), thresholds[ix])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}