ML_function.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# ML function for metabolome predictions using Random Forests"
      ],
      "metadata": {
        "id": "NMgpYVYoisFz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import subprocess\n",
        "import sys\n",
        "\n",
        "def install(package):\n",
        "    subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", package])\n",
        "\n",
        "install('dill')\n",
        "install('shap==0.37.0')\n",
        "install('scikit-learn-intelex')\n",
        "install('pandas')\n",
        "\n",
        "import pandas as pd\n",
        "import time\n",
        "import sklearn\n",
        "import numpy as np\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "from collections import Counter\n",
        "import scipy\n",
        "install('missingno')\n",
        "import missingno\n",
        "install('adjustText')\n",
        "from adjustText import adjust_text\n",
        "np.random.seed(123)\n",
        "import dill\n",
        "import math\n",
        "\n",
        "# machine learning\n",
        "# from sklearnex import patch_sklearn, unpatch_sklearn\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.multioutput import MultiOutputRegressor\n",
        "from sklearn.ensemble import RandomForestRegressor\n",
        "from sklearn.linear_model import Ridge\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.metrics import mean_squared_error\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.feature_selection import SelectKBest\n",
        "from multiprocessing import Pool"
      ],
      "metadata": {
        "id": "xL_j1L6cjOu6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#If running code on Google Colab, mount drive to access data files\n",
        "import os\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "#Name of folder that holds datasets\n",
        "file_folder = 'your_file_folder'\n",
        "file_path = '/content/drive/My\\ Drive/' + file_folder\n",
        "%cd {file_path}"
      ],
      "metadata": {
        "id": "5rcaqB_Qrk7g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Read in data (rows = cell lines, columns = features)\n",
        "input_features = pd.read_csv('./your_file_here.csv',index_col=0)\n",
        "metabolomics = pd.read_csv('./your_metabolomics_here.csv',index_col=0)"
      ],
      "metadata": {
        "id": "HkeOAmuUjwQc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Random Forests"
      ],
      "metadata": {
        "id": "SZ8Z5RgsUrCA"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PuAaod98iFJL"
      },
      "outputs": [],
      "source": [
        "scaler = StandardScaler()\n",
        "def train_multiRegressCV(classifier, data, x_cols, y_cols, n_splits=5, pval_cutoff=0.05, scale_y=True):\n",
        "\n",
        "  # split into train and test data: all metabolites, median data\n",
        "  X =  data.loc[:, x_cols]\n",
        "  y =  data.loc[:, y_cols]\n",
        "\n",
        "  # create multi-output ridge regression model\n",
        "  if classifier==\"RF\":\n",
        "    model=RandomForestRegressor(random_state=0, n_estimators=100, n_jobs=-1, max_depth=10)\n",
        "    print(\"Training model with random forest!\")\n",
        "\n",
        "  else:\n",
        "    print(\"Enter model type!\")\n",
        "    sys.exit()\n",
        "\n",
        "  clf = MultiOutputRegressor(model)\n",
        "\n",
        "  cv = KFold(n_splits=n_splits,\n",
        "                shuffle=True,\n",
        "                random_state=123)\n",
        "\n",
        "  # create empty lists to store CV scores, confusion mat, etc.\n",
        "  df_y_test_all = pd.DataFrame(columns=y_cols)\n",
        "  df_y_pred_all = pd.DataFrame(columns=y_cols)\n",
        "\n",
        "  count = 1\n",
        "\n",
        "  # loop through cross-val folds\n",
        "  all_means = pd.DataFrame(index=x_cols)\n",
        "\n",
        "  for train_index, test_index in cv.split(X, y):\n",
        "      start_time = time.perf_counter()\n",
        "      print(count,flush=True)\n",
        "      X_trainCV, X_testCV = X.iloc[train_index], X.iloc[test_index]\n",
        "      y_trainCV, y_testCV = y.iloc[train_index], y.iloc[test_index]\n",
        "\n",
        "      X_trainCV =  scaler.fit_transform(X_trainCV)\n",
        "      X_testCV =  scaler.transform(X_testCV)\n",
        "\n",
        "      if scale_y:\n",
        "        y_trainCV =  scaler.fit_transform(y_trainCV)\n",
        "        y_testCV =  scaler.transform(y_testCV)\n",
        "\n",
        "      tmp_mdl = clf.fit(X_trainCV, y_trainCV)\n",
        "      y_predCV = tmp_mdl.predict(X_testCV)\n",
        "\n",
        "      df_pred_temp  = pd.DataFrame(y_predCV, columns=y_cols)\n",
        "      df_test_temp = pd.DataFrame(y_testCV, columns=y_cols)\n",
        "\n",
        "      df_y_test_all = pd.concat([df_y_test_all, df_test_temp])\n",
        "      df_y_pred_all = pd.concat([df_y_pred_all, df_pred_temp])\n",
        "\n",
        "      count = count+1\n",
        "\n",
        "      finish_time = time.perf_counter()\n",
        "      print(\"CV fold finished in {} seconds\".format(finish_time-start_time),flush=True)\n",
        "\n",
        "  #print(\"Shape of y_test:\")\n",
        "  #print(df_y_test_all.shape)\n",
        "  #print(\"Calculating Pearson's R for each metabolite...\")\n",
        "  r = []\n",
        "  for i, col in enumerate(y_cols):\n",
        "    r.append(scipy.stats.pearsonr(df_y_test_all.iloc[:,i], df_y_pred_all.iloc[:,i]))\n",
        "\n",
        "  df_results = pd.DataFrame(r, columns=['R','pval'], index=y_cols)  # index=y_cols,\n",
        "\n",
        "  #print(\"Shape of results:\")\n",
        "  #print(df_results.shape)\n",
        "\n",
        "\n",
        "  df_results['Significant']  = (df_results.pval < pval_cutoff) & (df_results.R > 0)\n",
        "  df_results['R2']  = df_results.R**2\n",
        "\n",
        "  X_final = scaler.fit_transform(X)\n",
        "  X_final = pd.DataFrame(X_final,columns=X.columns,index=X.index)\n",
        "  if scale_y:\n",
        "    y_final = scaler.fit_transform(y)\n",
        "    y_final = pd.DataFrame(y_final,columns=y.columns,index=y.index)\n",
        "\n",
        "  print(\"Training final model on dataset of {} samples and {} features\".format(X_final.shape[0], X_final.shape[1]))\n",
        "\n",
        "  final_mdl = clf.fit(X_final, y_final)\n",
        "\n",
        "  return df_results, final_mdl"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Ridge and Lasso Regression"
      ],
      "metadata": {
        "id": "qJSvdqgUUtKw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "scaler = StandardScaler()\n",
        "def train_tune_CV(classifier,data, x_cols, y_cols,\n",
        "                  n_splits=5, pval_cutoff=0.05, alphas=[1e-2, 0.1, 1, 10, 100, 1000], scale_y=True):\n",
        "\n",
        "  # split into train and test data: all metabolites, median data\n",
        "  X =  data.loc[:, x_cols]\n",
        "  y =  data.loc[:, y_cols]\n",
        "\n",
        "  if classifier==\"Ridge\":\n",
        "    # create multi-output ridge regression model\n",
        "    model=Ridge(alpha=1.0, max_iter=None, tol=0.001, solver='auto', random_state=0)\n",
        "  elif classifier==\"Lasso\":\n",
        "    model=Lasso(alpha=1.0, max_iter=500, tol=0.001, random_state=0)\n",
        "\n",
        "  multi_ridge = MultiOutputRegressor(model)\n",
        "  hyperParameters = {'estimator__alpha':alphas}\n",
        "  gridSearch = GridSearchCV(multi_ridge, hyperParameters, scoring='r2', cv=5)\n",
        "\n",
        "  cv = KFold(n_splits=n_splits,\n",
        "                shuffle=True,\n",
        "                random_state=123)\n",
        "\n",
        "  # create empty lists to store CV scores, confusion mat, etc.\n",
        "  df_y_test_all = pd.DataFrame(columns=y_cols)\n",
        "  df_y_pred_all = pd.DataFrame(columns=y_cols)\n",
        "\n",
        "  count = 1\n",
        "\n",
        "  # loop through cross-val folds\n",
        "  all_means = pd.DataFrame(index=x_cols)\n",
        "\n",
        "  df_alpha_cv = pd.DataFrame()\n",
        "  df_r_cv = pd.DataFrame()\n",
        "\n",
        "  for count, (train_index, test_index) in enumerate(cv.split(X, y)):\n",
        "      print(count)\n",
        "      X_trainCV, X_testCV = X.iloc[train_index], X.iloc[test_index]\n",
        "      y_trainCV, y_testCV = y.iloc[train_index], y.iloc[test_index]\n",
        "\n",
        "      X_trainCV =  scaler.fit_transform(X_trainCV)\n",
        "      X_testCV =  scaler.transform(X_testCV)\n",
        "\n",
        "      if scale_y:\n",
        "        y_trainCV = scaler.fit_transform(y_trainCV)\n",
        "        y_testCV = scaler.transform(y_testCV)\n",
        "\n",
        "      #\n",
        "      tmp_mdl = gridSearch.fit(X_trainCV, y_trainCV)\n",
        "\n",
        "      # predict on CV-test set\n",
        "      y_predCV = tmp_mdl.predict(X_testCV)\n",
        "\n",
        "      # store tuned alpha values\n",
        "      my_alphas = []\n",
        "      r_cv = []\n",
        "\n",
        "      for i in range(len(tmp_mdl.best_estimator_.estimators_)):\n",
        "        my_alphas.append(tmp_mdl.best_estimator_.estimators_[i].get_params()['alpha'])\n",
        "        r_cv.append(scipy.stats.pearsonr(y_predCV[:,i], y_testCV[:,i])[0])\n",
        "\n",
        "      df_alpha_cv.loc[:,count] = my_alphas\n",
        "      df_r_cv.loc[:,count] = r_cv\n",
        "\n",
        "\n",
        "      # # calculate r2 for true vs predicted values of CV-test set\n",
        "      # tmp_scores = r2_score(y_testCV, y_predCV, multioutput='raw_values')\n",
        "\n",
        "      df_pred_temp  = pd.DataFrame(y_predCV, columns=y_cols)\n",
        "      df_test_temp = pd.DataFrame(y_testCV, columns=y_cols)\n",
        "\n",
        "      df_y_test_all = pd.concat([df_y_test_all, df_test_temp])\n",
        "      df_y_pred_all = pd.concat([df_y_pred_all, df_pred_temp])\n",
        "\n",
        "  print(\"Calculating Pearson's R for each metabolite...\")\n",
        "  r = []\n",
        "  for i, col in enumerate(y_cols):\n",
        "    r.append(scipy.stats.pearsonr(df_y_test_all.iloc[:,i], df_y_pred_all.iloc[:,i]))\n",
        "\n",
        "  df_results = pd.DataFrame(r, columns=['R','pval'], index=y_cols)  # index=y_cols,\n",
        "\n",
        "  df_results['Significant']  = (df_results.pval < pval_cutoff) & (df_results.R > 0)\n",
        "  df_results['R2']  = df_results.R**2\n",
        "\n",
        "  print(\"For each metabolite, find CV fold with best R and get alpha...\")\n",
        "  ix_best_fold = df_r_cv.idxmax(axis=1)\n",
        "  best_alphas = df_alpha_cv.values[np.arange(len(ix_best_fold)),ix_best_fold]\n",
        "  Counter(best_alphas)\n",
        "  #savetxt('best_alphas_phos.csv', best_alphas)\n",
        "\n",
        "  print(\"Training final models with best alphas...\")\n",
        "  X_final = scaler.fit_transform(X)\n",
        "  X_final = pd.DataFrame(X_final,columns=X.columns,index=X.index)\n",
        "  if scale_y:\n",
        "    y_final = scaler.fit_transform(y)\n",
        "    y_final = pd.DataFrame(y_final,columns=y.columns,index=y.index)\n",
        "\n",
        "  final_mdls = list()\n",
        "  for i in range(len(best_alphas)):\n",
        "    if classifier==\"Ridge\":\n",
        "        model=Ridge(alpha=best_alphas[i], max_iter=None, tol=0.001, solver='auto', random_state=0)\n",
        "    else:\n",
        "        model=Lasso(alpha=best_alphas[i], max_iter=500, tol=0.001, random_state=0)\n",
        "    model.fit(X_final.iloc[:,:], y_final.iloc[:,i])\n",
        "    final_mdls.append(model)\n",
        "  return df_results, final_mdls, best_alphas"
      ],
      "metadata": {
        "id": "GYgI-J2YUvig"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get feature names\n",
        "\n",
        "feature_names = input_features.columns\n",
        "\n",
        "my_metabs = metabolomics.columns\n",
        "\n",
        "feature_cell_lines = input_features.index\n",
        "\n",
        "my_metabs_cls = metabolomics.index"
      ],
      "metadata": {
        "id": "IejJszTpkKRr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# merge features+metabs by cell lines\n",
        "merged_data = pd.merge(input_features, metabolomics, left_index=True, right_index=True)\n",
        "print(merged_data.shape,flush=True)\n",
        "\n",
        "#%%\n",
        "print(\"METAB:\",flush=True)\n",
        "print(metabolomics.shape,flush=True)\n",
        "\n",
        "print(\"\\nFeats:\",flush=True)\n",
        "print(input_features.shape,flush=True)\n",
        "merged_feats_mets =  pd.merge(input_features, metabolomics, left_index=True, right_index=True)\n",
        "print(merged_feats_mets.shape,flush=True)"
      ],
      "metadata": {
        "id": "AF4QHTtTkcUj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Assign significance value (Bonferroni correction)\n",
        "pval_cutoff_ccle = 0.05 / len(my_metabs)"
      ],
      "metadata": {
        "id": "t8ChGdKyk1pT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "results_df, model = train_multiRegressCV(\"RF\",\n",
        "                                            data = merged_feats_mets,\n",
        "                                            x_cols = feature_names,\n",
        "                                            y_cols = my_metabs,\n",
        "                                            pval_cutoff = pval_cutoff_ccle,\n",
        "                                            n_splits = 5)\n",
        "\n",
        "print(\"Metabolites below Bonferroni pval cutoff: {}\".format(np.sum(results_df.Significant)),flush=True)\n",
        "\n",
        "\n",
        "#%%\n",
        "df = pd.DataFrame({\"pearsons_r\":results_df.R,\n",
        "                                \"model_pval\":results_df.pval,\n",
        "                                \"metabolite_significant\":results_df.Significant})\n",
        "df.to_csv(\"./your_results.csv\")\n",
        "\n",
        "# save models\n",
        "print(\"Saving models!\",flush=True)\n",
        "with open('./your_final_rf_model.pkl', 'wb') as f:\n",
        "   dill.dump(model, f)\n",
        "print(\"Done!\",flush=True)"
      ],
      "metadata": {
        "id": "-ImyA4F_lFQb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R4jt5pOOPhg5"
      },
      "source": [
        "# Example"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FzMInMOFPwkE"
      },
      "source": [
        "Using CCLE Histone PTM Data and CCLE Metabolomics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YEERKEmuh2mk"
      },
      "outputs": [],
      "source": [
        "scaler = StandardScaler()\n",
        "def train_multiRegressCV(classifier, data, x_cols, y_cols, n_splits=5, pval_cutoff=0.05, scale_y=True):\n",
        "\n",
        "  # split into train and test data: all metabolites, median data\n",
        "  X =  data.loc[:, x_cols]\n",
        "  y =  data.loc[:, y_cols]\n",
        "\n",
        "  # create multi-output ridge regression model\n",
        "  if classifier==\"RF\":\n",
        "    model=RandomForestRegressor(random_state=0, n_estimators=100, n_jobs=-1, max_depth=10)\n",
        "    print(\"Training model with random forest!\")\n",
        "\n",
        "  else:\n",
        "    print(\"Enter model type!\")\n",
        "    sys.exit()\n",
        "\n",
        "  clf = MultiOutputRegressor(model)\n",
        "\n",
        "  cv = KFold(n_splits=n_splits,\n",
        "                shuffle=True,\n",
        "                random_state=123)\n",
        "\n",
        "  # create empty lists to store CV scores, confusion mat, etc.\n",
        "  df_y_test_all = pd.DataFrame(columns=y_cols)\n",
        "  df_y_pred_all = pd.DataFrame(columns=y_cols)\n",
        "\n",
        "  count = 1\n",
        "\n",
        "  # loop through cross-val folds\n",
        "  all_means = pd.DataFrame(index=x_cols)\n",
        "\n",
        "  for train_index, test_index in cv.split(X, y):\n",
        "      start_time = time.perf_counter()\n",
        "      print(count,flush=True)\n",
        "      X_trainCV, X_testCV = X.iloc[train_index], X.iloc[test_index]\n",
        "      y_trainCV, y_testCV = y.iloc[train_index], y.iloc[test_index]\n",
        "\n",
        "      X_trainCV =  scaler.fit_transform(X_trainCV)\n",
        "      X_testCV =  scaler.transform(X_testCV)\n",
        "\n",
        "      if scale_y:\n",
        "        y_trainCV =  scaler.fit_transform(y_trainCV)\n",
        "        y_testCV =  scaler.transform(y_testCV)\n",
        "\n",
        "      tmp_mdl = clf.fit(X_trainCV, y_trainCV)\n",
        "      y_predCV = tmp_mdl.predict(X_testCV)\n",
        "\n",
        "      df_pred_temp  = pd.DataFrame(y_predCV, columns=y_cols)\n",
        "      df_test_temp = pd.DataFrame(y_testCV, columns=y_cols)\n",
        "\n",
        "      df_y_test_all = pd.concat([df_y_test_all, df_test_temp])\n",
        "      df_y_pred_all = pd.concat([df_y_pred_all, df_pred_temp])\n",
        "\n",
        "      count = count+1\n",
        "\n",
        "      finish_time = time.perf_counter()\n",
        "      print(\"CV fold finished in {} seconds\".format(finish_time-start_time),flush=True)\n",
        "\n",
        "  #print(\"Shape of y_test:\")\n",
        "  #print(df_y_test_all.shape)\n",
        "  #print(\"Calculating Pearson's R for each metabolite...\")\n",
        "  r = []\n",
        "  for i, col in enumerate(y_cols):\n",
        "    r.append(scipy.stats.pearsonr(df_y_test_all.iloc[:,i], df_y_pred_all.iloc[:,i]))\n",
        "\n",
        "  df_results = pd.DataFrame(r, columns=['R','pval'], index=y_cols)  # index=y_cols,\n",
        "\n",
        "  #print(\"Shape of results:\")\n",
        "  #print(df_results.shape)\n",
        "\n",
        "\n",
        "  df_results['Significant']  = (df_results.pval < pval_cutoff) & (df_results.R > 0)\n",
        "  df_results['R2']  = df_results.R**2\n",
        "\n",
        "  X_final = scaler.fit_transform(X)\n",
        "  X_final = pd.DataFrame(X_final,columns=X.columns,index=X.index)\n",
        "  if scale_y:\n",
        "    y_final = scaler.fit_transform(y)\n",
        "    y_final = pd.DataFrame(y_final,columns=y.columns,index=y.index)\n",
        "\n",
        "  print(\"Training final model on dataset of {} samples and {} features\".format(X_final.shape[0], X_final.shape[1]))\n",
        "\n",
        "  final_mdl = clf.fit(X_final, y_final)\n",
        "\n",
        "  return df_results, final_mdl"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jeCBuztsPtUd"
      },
      "outputs": [],
      "source": [
        "features_file_path = \"./CCLE_hist.csv\"\n",
        "metabolomics_file_path = \"./CCLE_metabolomics_averages.csv\"\n",
        "input_features = pd.read_csv(features_file_path, index_col=0)\n",
        "metabolomics = pd.read_csv(metabolomics_file_path, index_col=0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "O_8qL4_VgLya"
      },
      "outputs": [],
      "source": [
        "# Get feature names\n",
        "\n",
        "feature_names = input_features.columns\n",
        "\n",
        "my_metabs = metabolomics.columns\n",
        "\n",
        "feature_cell_lines = input_features.index\n",
        "\n",
        "my_metabs_cls = metabolomics.index"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-Ed8uOKIgR1P"
      },
      "outputs": [],
      "source": [
        "# merge features+metabs by cell lines\n",
        "merged_data = pd.merge(input_features, metabolomics, left_index=True, right_index=True)\n",
        "print(merged_data.shape,flush=True)\n",
        "\n",
        "#%%\n",
        "print(\"METAB:\",flush=True)\n",
        "print(metabolomics.shape,flush=True)\n",
        "\n",
        "print(\"\\nFeats:\",flush=True)\n",
        "print(input_features.shape,flush=True)\n",
        "merged_feats_mets =  pd.merge(input_features, metabolomics, left_index=True, right_index=True)\n",
        "print(merged_feats_mets.shape,flush=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "A5y8IysBhDZ3"
      },
      "outputs": [],
      "source": [
        "# Assign significance value (Bonferroni correction)\n",
        "pval_cutoff_ccle = 0.05 / len(my_metabs)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "PmhGw98EhFX_"
      },
      "outputs": [],
      "source": [
        "results_df, model = train_multiRegressCV(\"RF\",\n",
        "                                            data = merged_feats_mets,\n",
        "                                            x_cols = feature_names,\n",
        "                                            y_cols = my_metabs,\n",
        "                                            pval_cutoff = pval_cutoff_ccle,\n",
        "                                            n_splits = 5)\n",
        "\n",
        "print(\"Metabolites below Bonferroni pval cutoff: {}\".format(np.sum(results_df.Significant)),flush=True)\n",
        "\n",
        "\n",
        "#%%\n",
        "df = pd.DataFrame({\"pearsons_r\":results_df.R,\n",
        "                                \"model_pval\":results_df.pval,\n",
        "                                \"metabolite_significant\":results_df.Significant})\n",
        "df.to_csv(\"./CCLE_histone_results.csv\")\n",
        "\n",
        "# save models\n",
        "print(\"Saving models!\",flush=True)\n",
        "with open('./CCLE_histone_model.pkl', 'wb') as f:\n",
        "   dill.dump(model, f)\n",
        "print(\"Done!\",flush=True)"
      ]
    }
  ]
}