auto-ml.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Automated ML"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this file we use AutoML to search for the best model that predicts correctly if a couple will divorce or not based on the answer to the questions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "gather": {
     "logged": 1640553687760
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SDK version: 1.36.0\n"
     ]
    }
   ],
   "source": [
    "import logging\n",
    "import os\n",
    "import csv\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import datasets\n",
    "import pkg_resources\n",
    "\n",
    "import azureml.core\n",
    "from azureml.core.experiment import Experiment\n",
    "from azureml.core.workspace import Workspace\n",
    "from azureml.train.automl import AutoMLConfig\n",
    "from azureml.core.dataset import Dataset\n",
    "\n",
    "from azureml.pipeline.steps import AutoMLStep\n",
    "\n",
    "# Check core SDK version number\n",
    "print(\"SDK version:\", azureml.core.VERSION)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize Workspace\n",
    "Initialize a workspace object from persisted configuration at config.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "gather": {
     "logged": 1640553732577
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "quick-starts-ws-177093\n",
      "aml-quickstarts-177093\n",
      "southcentralus\n",
      "f5091c60-1c3c-430f-8d81-d802f6bf2414\n"
     ]
    }
   ],
   "source": [
    "ws = Workspace.from_config()\n",
    "print(ws.name, ws.resource_group, ws.location, ws.subscription_id, sep = '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create an Azure ML experiment\n",
    "Let's create an experiment\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "gather": {
     "logged": 1640553777201
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"width:100%\"><tr><th>Name</th><th>Workspace</th><th>Report Page</th><th>Docs Page</th></tr><tr><td>ml-experiment-1</td><td>quick-starts-ws-177093</td><td><a href=\"https://ml.azure.com/experiments/id/0a5b1e34-81c6-4489-ac01-58a7e09d1de4?wsid=/subscriptions/f5091c60-1c3c-430f-8d81-d802f6bf2414/resourcegroups/aml-quickstarts-177093/workspaces/quick-starts-ws-177093&amp;tid=660b3398-b80e-49d2-bc5b-ac1dc93b5254\" target=\"_blank\" rel=\"noopener\">Link to Azure Machine Learning studio</a></td><td><a href=\"https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.experiment.Experiment?view=azure-ml-py\" target=\"_blank\" rel=\"noopener\">Link to Documentation</a></td></tr></table>"
      ],
      "text/plain": [
       "Experiment(Name: ml-experiment-1,\n",
       "Workspace: quick-starts-ws-177093)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Choose a name for the run history container in the workspace.\n",
    "# NOTE: update these to match your existing experiment name\n",
    "experiment_name = 'ml-experiment-1'\n",
    "project_folder = './pipeline-project'\n",
    "\n",
    "experiment = Experiment(ws, experiment_name)\n",
    "experiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create or Attach an AmlCompute cluster\n",
    "Let's create a computer node if it doesn't exist or use the existing one."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "gather": {
     "logged": 1640554421840
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found existing cluster, use it.\n",
      "Succeeded............\n",
      "AmlCompute wait for completion finished\n",
      "\n",
      "Wait timeout has been reached\n",
      "Current provisioning state of AmlCompute is \"Succeeded\" and current node count is \"0\"\n"
     ]
    }
   ],
   "source": [
    "from azureml.core.compute import AmlCompute\n",
    "from azureml.core.compute import ComputeTarget\n",
    "from azureml.core.compute_target import ComputeTargetException\n",
    "\n",
    "# NOTE: update the cluster name to match the existing cluster\n",
    "# Choose a name for your CPU cluster\n",
    "amlcompute_cluster_name = \"cpuclusterautoml\"\n",
    "\n",
    "# Verify that cluster does not exist already\n",
    "try:\n",
    "    compute_target = ComputeTarget(workspace=ws, name=amlcompute_cluster_name)\n",
    "    print('Found existing cluster, use it.')\n",
    "except ComputeTargetException:\n",
    "    compute_config = AmlCompute.provisioning_configuration(vm_size='STANDARD_D2_V2',# for GPU, use \"STANDARD_NC6\"\n",
    "                                                           #vm_priority = 'lowpriority', # optional\n",
    "                                                           max_nodes=4)\n",
    "    compute_target = ComputeTarget.create(ws, amlcompute_cluster_name, compute_config)\n",
    "\n",
    "compute_target.wait_for_completion(show_output=True, min_node_count = 1, timeout_in_minutes = 1)\n",
    "# For a more detailed view of current AmlCompute status, use get_status()."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data\n",
    "The dataset consists on 54 features that corresponds to questions and 170 samples which corresponds to the answer of 170 couples to those 54 questions. The outcome is the divorced status of the couples based on their answers (1 divorced/0 not divorced)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "gather": {
     "logged": 1640554428138
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataset NOT found in workspace\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Q1</th>\n",
       "      <th>Q2</th>\n",
       "      <th>Q3</th>\n",
       "      <th>Q4</th>\n",
       "      <th>Q5</th>\n",
       "      <th>Q6</th>\n",
       "      <th>Q7</th>\n",
       "      <th>Q8</th>\n",
       "      <th>Q9</th>\n",
       "      <th>Q10</th>\n",
       "      <th>...</th>\n",
       "      <th>Q46</th>\n",
       "      <th>Q47</th>\n",
       "      <th>Q48</th>\n",
       "      <th>Q49</th>\n",
       "      <th>Q50</th>\n",
       "      <th>Q51</th>\n",
       "      <th>Q52</th>\n",
       "      <th>Q53</th>\n",
       "      <th>Q54</th>\n",
       "      <th>Divorce</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>170.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.776471</td>\n",
       "      <td>1.652941</td>\n",
       "      <td>1.764706</td>\n",
       "      <td>1.482353</td>\n",
       "      <td>1.541176</td>\n",
       "      <td>0.747059</td>\n",
       "      <td>0.494118</td>\n",
       "      <td>1.452941</td>\n",
       "      <td>1.458824</td>\n",
       "      <td>1.576471</td>\n",
       "      <td>...</td>\n",
       "      <td>2.552941</td>\n",
       "      <td>2.270588</td>\n",
       "      <td>2.741176</td>\n",
       "      <td>2.382353</td>\n",
       "      <td>2.429412</td>\n",
       "      <td>2.476471</td>\n",
       "      <td>2.517647</td>\n",
       "      <td>2.241176</td>\n",
       "      <td>2.011765</td>\n",
       "      <td>0.494118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.627257</td>\n",
       "      <td>1.468654</td>\n",
       "      <td>1.415444</td>\n",
       "      <td>1.504327</td>\n",
       "      <td>1.632169</td>\n",
       "      <td>0.904046</td>\n",
       "      <td>0.898698</td>\n",
       "      <td>1.546371</td>\n",
       "      <td>1.557976</td>\n",
       "      <td>1.421529</td>\n",
       "      <td>...</td>\n",
       "      <td>1.371786</td>\n",
       "      <td>1.586841</td>\n",
       "      <td>1.137348</td>\n",
       "      <td>1.511587</td>\n",
       "      <td>1.405090</td>\n",
       "      <td>1.260238</td>\n",
       "      <td>1.476537</td>\n",
       "      <td>1.505634</td>\n",
       "      <td>1.667611</td>\n",
       "      <td>0.501442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               Q1          Q2          Q3          Q4          Q5          Q6  \\\n",
       "count  170.000000  170.000000  170.000000  170.000000  170.000000  170.000000   \n",
       "mean     1.776471    1.652941    1.764706    1.482353    1.541176    0.747059   \n",
       "std      1.627257    1.468654    1.415444    1.504327    1.632169    0.904046   \n",
       "min      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "25%      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "50%      2.000000    2.000000    2.000000    1.000000    1.000000    0.000000   \n",
       "75%      3.000000    3.000000    3.000000    3.000000    3.000000    1.000000   \n",
       "max      4.000000    4.000000    4.000000    4.000000    4.000000    4.000000   \n",
       "\n",
       "               Q7          Q8          Q9         Q10  ...         Q46  \\\n",
       "count  170.000000  170.000000  170.000000  170.000000  ...  170.000000   \n",
       "mean     0.494118    1.452941    1.458824    1.576471  ...    2.552941   \n",
       "std      0.898698    1.546371    1.557976    1.421529  ...    1.371786   \n",
       "min      0.000000    0.000000    0.000000    0.000000  ...    0.000000   \n",
       "25%      0.000000    0.000000    0.000000    0.000000  ...    2.000000   \n",
       "50%      0.000000    1.000000    1.000000    2.000000  ...    3.000000   \n",
       "75%      1.000000    3.000000    3.000000    3.000000  ...    4.000000   \n",
       "max      4.000000    4.000000    4.000000    4.000000  ...    4.000000   \n",
       "\n",
       "              Q47         Q48         Q49         Q50         Q51         Q52  \\\n",
       "count  170.000000  170.000000  170.000000  170.000000  170.000000  170.000000   \n",
       "mean     2.270588    2.741176    2.382353    2.429412    2.476471    2.517647   \n",
       "std      1.586841    1.137348    1.511587    1.405090    1.260238    1.476537   \n",
       "min      0.000000    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "25%      1.000000    2.000000    1.000000    1.000000    2.000000    1.000000   \n",
       "50%      2.000000    3.000000    3.000000    2.000000    3.000000    3.000000   \n",
       "75%      4.000000    4.000000    4.000000    4.000000    4.000000    4.000000   \n",
       "max      4.000000    4.000000    4.000000    4.000000    4.000000    4.000000   \n",
       "\n",
       "              Q53         Q54     Divorce  \n",
       "count  170.000000  170.000000  170.000000  \n",
       "mean     2.241176    2.011765    0.494118  \n",
       "std      1.505634    1.667611    0.501442  \n",
       "min      0.000000    0.000000    0.000000  \n",
       "25%      1.000000    0.000000    0.000000  \n",
       "50%      2.000000    2.000000    0.000000  \n",
       "75%      4.000000    4.000000    1.000000  \n",
       "max      4.000000    4.000000    1.000000  \n",
       "\n",
       "[8 rows x 55 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Try to load the dataset from the Workspace. Otherwise, create it from the file\n",
    "# NOTE: update the key to match the dataset name\n",
    "found = False\n",
    "key = \"divorce-ds\"\n",
    "description_text = \"\"\n",
    "\n",
    "if key in ws.datasets.keys(): \n",
    "        found = True\n",
    "        dataset = ws.datasets[key]\n",
    "        print('dataset found in workspace')\n",
    "\n",
    "if not found:\n",
    "        print('dataset NOT found in workspace')\n",
    "        # Create AML Dataset and register it into Workspace\n",
    "        example_data = 'https://raw.githubusercontent.com/joangerard/project-azure-piepline/master/divorce_data.csv'\n",
    "        dataset = Dataset.Tabular.from_delimited_files(example_data, separator=';')        \n",
    "        #Register Dataset in Workspace\n",
    "        dataset = dataset.register(workspace=ws,\n",
    "                                   name=key,\n",
    "                                   description=description_text)\n",
    "\n",
    "\n",
    "df = dataset.to_pandas_dataframe()\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Review the Dataset Result\n",
    "Here we can see the first 5 samples which corresponds to couples who divorced."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "gather": {
     "logged": 1640554441126
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Q1</th>\n",
       "      <th>Q2</th>\n",
       "      <th>Q3</th>\n",
       "      <th>Q4</th>\n",
       "      <th>Q5</th>\n",
       "      <th>Q6</th>\n",
       "      <th>Q7</th>\n",
       "      <th>Q8</th>\n",
       "      <th>Q9</th>\n",
       "      <th>Q10</th>\n",
       "      <th>...</th>\n",
       "      <th>Q46</th>\n",
       "      <th>Q47</th>\n",
       "      <th>Q48</th>\n",
       "      <th>Q49</th>\n",
       "      <th>Q50</th>\n",
       "      <th>Q51</th>\n",
       "      <th>Q52</th>\n",
       "      <th>Q53</th>\n",
       "      <th>Q54</th>\n",
       "      <th>Divorce</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 55 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Q1  Q2  Q3  Q4  Q5  Q6  Q7  Q8  Q9  Q10  ...  Q46  Q47  Q48  Q49  Q50  Q51  \\\n",
       "0   2   2   4   1   0   0   0   0   0    0  ...    2    1    3    3    3    2   \n",
       "1   4   4   4   4   4   0   0   4   4    4  ...    2    2    3    4    4    4   \n",
       "2   2   2   2   2   1   3   2   1   1    2  ...    3    2    3    1    1    1   \n",
       "3   3   2   3   2   3   3   3   3   3    3  ...    2    2    3    3    3    3   \n",
       "4   2   2   1   1   1   1   0   0   0    0  ...    2    1    2    3    2    2   \n",
       "\n",
       "   Q52  Q53  Q54  Divorce  \n",
       "0    3    2    1        1  \n",
       "1    4    2    2        1  \n",
       "2    2    2    2        1  \n",
       "3    2    2    2        1  \n",
       "4    2    1    0        1  \n",
       "\n",
       "[5 rows x 55 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.take(5).to_pandas_dataframe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AutoML Configuration\n",
    "This creates an AutoML settings object and it indicates the label column to be Divorce."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "gather": {
     "logged": 1640554446811
    }
   },
   "outputs": [],
   "source": [
    "automl_settings = {\n",
    "    \"experiment_timeout_minutes\": 20,\n",
    "    \"max_concurrent_iterations\": 5,\n",
    "    \"primary_metric\" : 'accuracy'\n",
    "}\n",
    "automl_config = AutoMLConfig(compute_target=compute_target,\n",
    "                             task = \"classification\",\n",
    "                             training_data=dataset,\n",
    "                             label_column_name=\"Divorce\",   \n",
    "                             path = project_folder,\n",
    "                             enable_early_stopping= True,\n",
    "                             featurization= 'auto',\n",
    "                             debug_log = \"automl_errors.log\",\n",
    "                             **automl_settings\n",
    "                            )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create Pipeline and AutoMLStep\n",
    "Here we will use a pipeline to train the data. It will output some metrics and the best model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "gather": {
     "logged": 1640554482199
    }
   },
   "outputs": [],
   "source": [
    "from azureml.pipeline.core import PipelineData, TrainingOutput\n",
    "\n",
    "ds = ws.get_default_datastore()\n",
    "metrics_output_name = 'metrics_output'\n",
    "best_model_output_name = 'best_model_output'\n",
    "\n",
    "metrics_data = PipelineData(name='metrics_data',\n",
    "                           datastore=ds,\n",
    "                           pipeline_output_name=metrics_output_name,\n",
    "                           training_output=TrainingOutput(type='Metrics'))\n",
    "model_data = PipelineData(name='model_data',\n",
    "                           datastore=ds,\n",
    "                           pipeline_output_name=best_model_output_name,\n",
    "                           training_output=TrainingOutput(type='Model'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create an AutoMLStep."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "gather": {
     "logged": 1640554491545
    },
    "tags": [
     "automlstep-remarks-sample1"
    ]
   },
   "outputs": [],
   "source": [
    "automl_step = AutoMLStep(\n",
    "    name='automl_module',\n",
    "    automl_config=automl_config,\n",
    "    outputs=[metrics_data, model_data],\n",
    "    allow_reuse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "gather": {
     "logged": 1640554503271
    },
    "tags": [
     "automlstep-remarks-sample2"
    ]
   },
   "outputs": [],
   "source": [
    "from azureml.pipeline.core import Pipeline\n",
    "pipeline = Pipeline(\n",
    "    description=\"pipeline_with_automlstep\",\n",
    "    workspace=ws,    \n",
    "    steps=[automl_step])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "gather": {
     "logged": 1640554512327
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created step automl_module [44196ea0][da4086c8-fce6-4f58-ae1e-8f22d6366f37], (This step will run and generate new outputs)\n",
      "Submitted PipelineRun 34d55c16-a260-4fbe-b524-5f156e0f34b4\n",
      "Link to Azure Machine Learning Portal: https://ml.azure.com/runs/34d55c16-a260-4fbe-b524-5f156e0f34b4?wsid=/subscriptions/f5091c60-1c3c-430f-8d81-d802f6bf2414/resourcegroups/aml-quickstarts-177093/workspaces/quick-starts-ws-177093&tid=660b3398-b80e-49d2-bc5b-ac1dc93b5254\n"
     ]
    }
   ],
   "source": [
    "pipeline_run = experiment.submit(pipeline)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "gather": {
     "logged": 1640554517417
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54685dec403247f99af60bbcb9fe439e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "_PipelineWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO', …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/aml.mini.widget.v1": "{\"status\": \"Completed\", \"workbench_run_details_uri\": \"https://ml.azure.com/runs/34d55c16-a260-4fbe-b524-5f156e0f34b4?wsid=/subscriptions/f5091c60-1c3c-430f-8d81-d802f6bf2414/resourcegroups/aml-quickstarts-177093/workspaces/quick-starts-ws-177093&tid=660b3398-b80e-49d2-bc5b-ac1dc93b5254\", \"run_id\": \"34d55c16-a260-4fbe-b524-5f156e0f34b4\", \"run_properties\": {\"run_id\": \"34d55c16-a260-4fbe-b524-5f156e0f34b4\", \"created_utc\": \"2022-01-20T13:33:28.624638Z\", \"properties\": {\"azureml.runsource\": \"azureml.PipelineRun\", \"runSource\": \"SDK\", \"runType\": \"SDK\", \"azureml.parameters\": \"{}\", \"azureml.continue_on_step_failure\": \"False\", \"azureml.pipelineComponent\": \"pipelinerun\"}, \"tags\": {}, \"end_time_utc\": \"2022-01-20T13:57:03.402373Z\", \"status\": \"Completed\", \"log_files\": {\"logs/azureml/executionlogs.txt\": \"https://mlstrg177093.blob.core.windows.net/azureml/ExperimentRun/dcid.34d55c16-a260-4fbe-b524-5f156e0f34b4/logs/azureml/executionlogs.txt?sv=2019-07-07&sr=b&sig=d7Kd3GRGl779abUzaYVvPIgDD3xyCtGS9dda5v8wWzY%3D&skoid=81bd2a5e-3ffb-475f-bbc3-669384e73454&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2022-01-20T13%3A05%3A58Z&ske=2022-01-21T21%3A15%3A58Z&sks=b&skv=2019-07-07&st=2022-01-20T15%3A03%3A18Z&se=2022-01-20T23%3A13%3A18Z&sp=r\", \"logs/azureml/stderrlogs.txt\": \"https://mlstrg177093.blob.core.windows.net/azureml/ExperimentRun/dcid.34d55c16-a260-4fbe-b524-5f156e0f34b4/logs/azureml/stderrlogs.txt?sv=2019-07-07&sr=b&sig=%2B4XmGrUqDyGKyeluoyfrfk2DuXC9Ibt5FaGcnSBosN4%3D&skoid=81bd2a5e-3ffb-475f-bbc3-669384e73454&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2022-01-20T13%3A05%3A58Z&ske=2022-01-21T21%3A15%3A58Z&sks=b&skv=2019-07-07&st=2022-01-20T15%3A03%3A18Z&se=2022-01-20T23%3A13%3A18Z&sp=r\", \"logs/azureml/stdoutlogs.txt\": \"https://mlstrg177093.blob.core.windows.net/azureml/ExperimentRun/dcid.34d55c16-a260-4fbe-b524-5f156e0f34b4/logs/azureml/stdoutlogs.txt?sv=2019-07-07&sr=b&sig=YVUBlG%2FpnqrQ9NLGnJqOySIhATi%2FHZZII%2F3CnjUe7WA%3D&skoid=81bd2a5e-3ffb-475f-bbc3-669384e73454&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2022-01-20T13%3A05%3A58Z&ske=2022-01-21T21%3A15%3A58Z&sks=b&skv=2019-07-07&st=2022-01-20T15%3A03%3A18Z&se=2022-01-20T23%3A13%3A18Z&sp=r\"}, \"log_groups\": [[\"logs/azureml/executionlogs.txt\", \"logs/azureml/stderrlogs.txt\", \"logs/azureml/stdoutlogs.txt\"]], \"run_duration\": \"0:23:34\", \"run_number\": \"1\", \"run_queued_details\": {\"status\": \"Finished\", \"details\": null}}, \"child_runs\": [{\"run_id\": \"296c76e2-158b-4482-99e3-b14ef348bc65\", \"name\": \"automl_module\", \"status\": \"Finished\", \"start_time\": \"2022-01-20T13:33:46.515117Z\", \"created_time\": \"2022-01-20T13:33:31.428096Z\", \"end_time\": \"2022-01-20T13:56:17.308528Z\", \"duration\": \"0:22:45\", \"run_number\": 2, \"metric\": null, \"run_type\": \"azureml.StepRun\", \"training_percent\": null, \"created_time_dt\": \"2022-01-20T13:33:31.428096Z\", \"is_reused\": \"\"}], \"children_metrics\": {\"categories\": null, \"series\": null, \"metricName\": null}, \"run_metrics\": [], \"run_logs\": \"[2022-01-20 13:33:31Z] Submitting 1 runs, first five are: 44196ea0:296c76e2-158b-4482-99e3-b14ef348bc65\\n[2022-01-20 13:57:03Z] Completing processing run id 296c76e2-158b-4482-99e3-b14ef348bc65.\\n\\nRun is completed.\", \"graph\": {\"datasource_nodes\": {\"48f6949d\": {\"node_id\": \"48f6949d\", \"name\": \"divorce-ds\"}}, \"module_nodes\": {\"44196ea0\": {\"node_id\": \"44196ea0\", \"name\": \"automl_module\", \"status\": \"Finished\", \"_is_reused\": false, \"run_id\": \"296c76e2-158b-4482-99e3-b14ef348bc65\"}}, \"edges\": [{\"source_node_id\": \"48f6949d\", \"source_node_name\": \"divorce-ds\", \"source_name\": \"data\", \"target_name\": \"training_data\", \"dst_node_id\": \"44196ea0\", \"dst_node_name\": \"automl_module\"}], \"child_runs\": [{\"run_id\": \"296c76e2-158b-4482-99e3-b14ef348bc65\", \"name\": \"automl_module\", \"status\": \"Finished\", \"start_time\": \"2022-01-20T13:33:46.515117Z\", \"created_time\": \"2022-01-20T13:33:31.428096Z\", \"end_time\": \"2022-01-20T13:56:17.308528Z\", \"duration\": \"0:22:45\", \"run_number\": 2, \"metric\": null, \"run_type\": \"azureml.StepRun\", \"training_percent\": null, \"created_time_dt\": \"2022-01-20T13:33:31.428096Z\", \"is_reused\": \"\"}]}, \"widget_settings\": {\"childWidgetDisplay\": \"popup\", \"send_telemetry\": false, \"log_level\": \"INFO\", \"sdk_version\": \"1.36.0\"}, \"loading\": false}"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from azureml.widgets import RunDetails\n",
    "RunDetails(pipeline_run).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "gather": {
     "logged": 1640555522651
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PipelineRunId: 34d55c16-a260-4fbe-b524-5f156e0f34b4\n",
      "Link to Azure Machine Learning Portal: https://ml.azure.com/runs/34d55c16-a260-4fbe-b524-5f156e0f34b4?wsid=/subscriptions/f5091c60-1c3c-430f-8d81-d802f6bf2414/resourcegroups/aml-quickstarts-177093/workspaces/quick-starts-ws-177093&tid=660b3398-b80e-49d2-bc5b-ac1dc93b5254\n",
      "PipelineRun Status: Running\n",
      "\n",
      "\n",
      "StepRunId: 296c76e2-158b-4482-99e3-b14ef348bc65\n",
      "Link to Azure Machine Learning Portal: https://ml.azure.com/runs/296c76e2-158b-4482-99e3-b14ef348bc65?wsid=/subscriptions/f5091c60-1c3c-430f-8d81-d802f6bf2414/resourcegroups/aml-quickstarts-177093/workspaces/quick-starts-ws-177093&tid=660b3398-b80e-49d2-bc5b-ac1dc93b5254\n",
      "StepRun( automl_module ) Status: Running\n",
      "\n",
      "StepRun(automl_module) Execution Summary\n",
      "=========================================\n",
      "StepRun( automl_module ) Status: Finished\n",
      "\n",
      "Warnings:\n",
      "No scores improved over last 20 iterations, so experiment stopped early. This early stopping behavior can be disabled by setting enable_early_stopping = False in AutoMLConfig for notebook/python SDK runs.\n",
      "{'runId': '296c76e2-158b-4482-99e3-b14ef348bc65', 'target': 'cpuclusterautoml', 'status': 'Completed', 'startTimeUtc': '2022-01-20T13:33:46.515117Z', 'endTimeUtc': '2022-01-20T13:56:17.308528Z', 'services': {}, 'warnings': [{'source': 'JasmineService', 'message': 'No scores improved over last 20 iterations, so experiment stopped early. This early stopping behavior can be disabled by setting enable_early_stopping = False in AutoMLConfig for notebook/python SDK runs.'}], 'properties': {'ContentSnapshotId': 'fc92b9ab-cdfa-41e5-8da8-d0ceb25267cb', 'StepType': 'AutoMLStep', 'azureml.moduleid': 'da4086c8-fce6-4f58-ae1e-8f22d6366f37', 'azureml.moduleName': 'automl_module', 'azureml.runsource': 'azureml.StepRun', 'azureml.nodeid': '44196ea0', 'azureml.pipelinerunid': '34d55c16-a260-4fbe-b524-5f156e0f34b4', 'azureml.pipeline': '34d55c16-a260-4fbe-b524-5f156e0f34b4', 'azureml.pipelineComponent': 'masterautomlcloud', 'num_iterations': '1000', 'training_type': 'TrainFull', 'acquisition_function': 'EI', 'metrics': 'accuracy', 'primary_metric': 'accuracy', 'train_split': '0', 'MaxTimeSeconds': None, 'acquisition_parameter': '0', 'num_cross_validation': None, 'target': 'cpuclusterautoml', 'RawAMLSettingsString': None, 'AMLSettingsJsonString': '{\"path\": null, \"name\": \"placeholder\", \"subscription_id\": \"f5091c60-1c3c-430f-8d81-d802f6bf2414\", \"resource_group\": \"aml-quickstarts-177093\", \"workspace_name\": \"quick-starts-ws-177093\", \"region\": \"southcentralus\", \"compute_target\": \"cpuclusterautoml\", \"spark_service\": null, \"azure_service\": null, \"many_models\": false, \"pipeline_fetch_max_batch_size\": 1, \"enable_batch_run\": false, \"enable_run_restructure\": false, \"start_auxiliary_runs_before_parent_complete\": false, \"enable_code_generation\": false, \"iterations\": 1000, \"primary_metric\": \"accuracy\", \"task_type\": \"classification\", \"positive_label\": null, \"data_script\": null, \"test_size\": 0.0, \"test_include_predictions_only\": false, \"validation_size\": 0.0, \"n_cross_validations\": null, \"y_min\": null, \"y_max\": null, \"num_classes\": null, \"featurization\": \"auto\", \"_ignore_package_version_incompatibilities\": false, \"is_timeseries\": false, \"max_cores_per_iteration\": 1, \"max_concurrent_iterations\": 5, \"iteration_timeout_minutes\": null, \"mem_in_mb\": null, \"enforce_time_on_windows\": false, \"experiment_timeout_minutes\": 20, \"experiment_exit_score\": null, \"whitelist_models\": null, \"blacklist_algos\": null, \"supported_models\": [\"RandomForest\", \"SGD\", \"KNN\", \"XGBoostClassifier\", \"GradientBoosting\", \"ExtremeRandomTrees\", \"LinearSVM\", \"TensorFlowLinearClassifier\", \"SVM\", \"LightGBM\", \"AveragedPerceptronClassifier\", \"LogisticRegression\", \"MultinomialNaiveBayes\", \"DecisionTree\", \"BernoulliNaiveBayes\", \"TensorFlowDNN\"], \"private_models\": [\"TabnetClassifier\"], \"auto_blacklist\": true, \"blacklist_samples_reached\": false, \"exclude_nan_labels\": true, \"verbosity\": 20, \"_debug_log\": \"automl_errors.log\", \"show_warnings\": false, \"model_explainability\": true, \"service_url\": null, \"sdk_url\": null, \"sdk_packages\": null, \"enable_onnx_compatible_models\": false, \"enable_split_onnx_featurizer_estimator_models\": false, \"vm_type\": \"STANDARD_D2_V2\", \"telemetry_verbosity\": 20, \"send_telemetry\": true, \"enable_dnn\": false, \"scenario\": \"SDK-1.13.0\", \"environment_label\": null, \"save_mlflow\": false, \"enable_categorical_indicators\": false, \"force_text_dnn\": false, \"enable_feature_sweeping\": true, \"enable_early_stopping\": true, \"early_stopping_n_iters\": 10, \"arguments\": null, \"dataset_id\": null, \"hyperdrive_config\": null, \"validation_dataset_id\": null, \"run_source\": null, \"metrics\": null, \"enable_metric_confidence\": false, \"enable_ensembling\": true, \"enable_stack_ensembling\": true, \"ensemble_iterations\": 15, \"enable_tf\": false, \"enable_subsampling\": null, \"subsample_seed\": null, \"enable_nimbusml\": false, \"enable_streaming\": false, \"force_streaming\": false, \"track_child_runs\": true, \"allowed_private_models\": [], \"label_column_name\": \"Divorce\", \"weight_column_name\": null, \"cv_split_column_names\": null, \"enable_local_managed\": false, \"_local_managed_run_id\": null, \"cost_mode\": 1, \"lag_length\": 0, \"metric_operation\": \"maximize\", \"preprocess\": true}', 'DataPrepJsonString': '{\\\\\"training_data\\\\\": {\\\\\"datasetId\\\\\": \\\\\"86f2404a-3c24-4d06-92a1-d7129686864c\\\\\"}, \\\\\"datasets\\\\\": 0}', 'EnableSubsampling': 'False', 'runTemplate': 'AutoML', 'Orchestrator': 'automl', 'ClientType': 'Others', '_aml_system_scenario_identification': 'Remote.Parent', 'root_attribution': 'azureml.StepRun', 'snapshotId': 'fc92b9ab-cdfa-41e5-8da8-d0ceb25267cb', 'SetupRunId': '296c76e2-158b-4482-99e3-b14ef348bc65_setup', 'SetupRunContainerId': 'dcid.296c76e2-158b-4482-99e3-b14ef348bc65_setup', 'ClientSdkVersion': '1.37.0', 'FeaturizationRunJsonPath': 'featurizer_container.json', 'FeaturizationRunId': '296c76e2-158b-4482-99e3-b14ef348bc65_featurize', 'ProblemInfoJsonString': '{\"dataset_num_categorical\": 0, \"is_sparse\": true, \"subsampling\": false, \"has_extra_col\": true, \"dataset_classes\": 2, \"dataset_features\": 270, \"dataset_samples\": 170, \"single_frequency_class_detected\": false}', 'ModelExplainRunId': '296c76e2-158b-4482-99e3-b14ef348bc65_ModelExplain'}, 'inputDatasets': [], 'outputDatasets': [], 'logFiles': {'logs/azureml/executionlogs.txt': 'https://mlstrg177093.blob.core.windows.net/azureml/ExperimentRun/dcid.296c76e2-158b-4482-99e3-b14ef348bc65/logs/azureml/executionlogs.txt?sv=2019-07-07&sr=b&sig=FoVaaqT2H98sR2PsXXZWmJkUjzGoGgdj6IFvUy8GqP0%3D&skoid=81bd2a5e-3ffb-475f-bbc3-669384e73454&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2022-01-20T13%3A05%3A58Z&ske=2022-01-21T21%3A15%3A58Z&sks=b&skv=2019-07-07&st=2022-01-20T13%3A43%3A47Z&se=2022-01-20T21%3A53%3A47Z&sp=r', 'logs/azureml/stderrlogs.txt': 'https://mlstrg177093.blob.core.windows.net/azureml/ExperimentRun/dcid.296c76e2-158b-4482-99e3-b14ef348bc65/logs/azureml/stderrlogs.txt?sv=2019-07-07&sr=b&sig=LyKYOEy40lgVh9uq2i3jWT9vhxZo74QVtg0ISAKcs6M%3D&skoid=81bd2a5e-3ffb-475f-bbc3-669384e73454&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2022-01-20T13%3A05%3A58Z&ske=2022-01-21T21%3A15%3A58Z&sks=b&skv=2019-07-07&st=2022-01-20T13%3A43%3A47Z&se=2022-01-20T21%3A53%3A47Z&sp=r', 'logs/azureml/stdoutlogs.txt': 'https://mlstrg177093.blob.core.windows.net/azureml/ExperimentRun/dcid.296c76e2-158b-4482-99e3-b14ef348bc65/logs/azureml/stdoutlogs.txt?sv=2019-07-07&sr=b&sig=zVdwZt%2BEisdeRtiF%2FIw35nf84yxyV%2FGQ%2FBH3w3v8XcQ%3D&skoid=81bd2a5e-3ffb-475f-bbc3-669384e73454&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2022-01-20T13%3A05%3A58Z&ske=2022-01-21T21%3A15%3A58Z&sks=b&skv=2019-07-07&st=2022-01-20T13%3A43%3A47Z&se=2022-01-20T21%3A53%3A47Z&sp=r'}, 'submittedBy': 'ODL_User 177093'}\n",
      "\n",
      "\n",
      "\n",
      "PipelineRun Execution Summary\n",
      "==============================\n",
      "PipelineRun Status: Finished\n",
      "{'runId': '34d55c16-a260-4fbe-b524-5f156e0f34b4', 'status': 'Completed', 'startTimeUtc': '2022-01-20T13:33:30.01353Z', 'endTimeUtc': '2022-01-20T13:57:03.402373Z', 'services': {}, 'properties': {'azureml.runsource': 'azureml.PipelineRun', 'runSource': 'SDK', 'runType': 'SDK', 'azureml.parameters': '{}', 'azureml.continue_on_step_failure': 'False', 'azureml.pipelineComponent': 'pipelinerun'}, 'inputDatasets': [], 'outputDatasets': [], 'logFiles': {'logs/azureml/executionlogs.txt': 'https://mlstrg177093.blob.core.windows.net/azureml/ExperimentRun/dcid.34d55c16-a260-4fbe-b524-5f156e0f34b4/logs/azureml/executionlogs.txt?sv=2019-07-07&sr=b&sig=uteo8acTwnkkGLXaDPi9%2FRl6Hpzd5W8%2BqPlaBWW73co%3D&skoid=81bd2a5e-3ffb-475f-bbc3-669384e73454&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2022-01-20T13%3A05%3A58Z&ske=2022-01-21T21%3A15%3A58Z&sks=b&skv=2019-07-07&st=2022-01-20T13%3A44%3A55Z&se=2022-01-20T21%3A54%3A55Z&sp=r', 'logs/azureml/stderrlogs.txt': 'https://mlstrg177093.blob.core.windows.net/azureml/ExperimentRun/dcid.34d55c16-a260-4fbe-b524-5f156e0f34b4/logs/azureml/stderrlogs.txt?sv=2019-07-07&sr=b&sig=cA%2Bkgr2kEdjWF6ReqEv0O6jwHwY2%2F5XXjRYtaIwgTlE%3D&skoid=81bd2a5e-3ffb-475f-bbc3-669384e73454&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2022-01-20T13%3A05%3A58Z&ske=2022-01-21T21%3A15%3A58Z&sks=b&skv=2019-07-07&st=2022-01-20T13%3A44%3A55Z&se=2022-01-20T21%3A54%3A55Z&sp=r', 'logs/azureml/stdoutlogs.txt': 'https://mlstrg177093.blob.core.windows.net/azureml/ExperimentRun/dcid.34d55c16-a260-4fbe-b524-5f156e0f34b4/logs/azureml/stdoutlogs.txt?sv=2019-07-07&sr=b&sig=PjdstqoUHjGEhNB0M7zFNWT3R55A7NpsZgN%2Bwn4GQwQ%3D&skoid=81bd2a5e-3ffb-475f-bbc3-669384e73454&sktid=660b3398-b80e-49d2-bc5b-ac1dc93b5254&skt=2022-01-20T13%3A05%3A58Z&ske=2022-01-21T21%3A15%3A58Z&sks=b&skv=2019-07-07&st=2022-01-20T13%3A44%3A55Z&se=2022-01-20T21%3A54%3A55Z&sp=r'}, 'submittedBy': 'ODL_User 177093'}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Finished'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline_run.wait_for_completion()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retrieve the metrics of all child runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "gather": {
     "logged": 1640555577091
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading azureml/296c76e2-158b-4482-99e3-b14ef348bc65/metrics_data\n",
      "Downloaded azureml/296c76e2-158b-4482-99e3-b14ef348bc65/metrics_data, 1 files out of an estimated total of 1\n"
     ]
    }
   ],
   "source": [
    "metrics_output = pipeline_run.get_pipeline_output(metrics_output_name)\n",
    "num_file_downloaded = metrics_output.download('.', show_progress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "gather": {
     "logged": 1640555588906
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_1</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_3</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_4</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_0</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_15</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_18</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_10</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_12</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_13</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_14</th>\n",
       "      <th>...</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_7</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_6</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_11</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_16</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_33</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_24</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_20</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_30</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_39</th>\n",
       "      <th>296c76e2-158b-4482-99e3-b14ef348bc65_27</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>average_precision_score_macro</th>\n",
       "      <td>[0.9967628205128205]</td>\n",
       "      <td>[0.99875]</td>\n",
       "      <td>[0.99875]</td>\n",
       "      <td>[0.9967628205128205]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.99875]</td>\n",
       "      <td>[0.99875]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9951781705948374]</td>\n",
       "      <td>...</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9967628205128205]</td>\n",
       "      <td>[0.99875]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.996475122100122]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.996475122100122]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9980128205128205]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1_score_micro</th>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9470588235294117]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "      <td>[0.9352941176470588]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>balanced_accuracy</th>\n",
       "      <td>[0.9684722222222222]</td>\n",
       "      <td>[0.9784722222222223]</td>\n",
       "      <td>[0.9684722222222222]</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9684722222222222]</td>\n",
       "      <td>[0.9684722222222222]</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9448611111111112]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9684722222222222]</td>\n",
       "      <td>[0.9684722222222222]</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9802777777777777]</td>\n",
       "      <td>[0.9349999999999999]</td>\n",
       "      <td>[0.9740277777777777]</td>\n",
       "      <td>[0.9802777777777777]</td>\n",
       "      <td>[0.9802777777777777]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>norm_macro_recall</th>\n",
       "      <td>[0.9369444444444444]</td>\n",
       "      <td>[0.9569444444444445]</td>\n",
       "      <td>[0.9369444444444444]</td>\n",
       "      <td>[0.9494444444444443]</td>\n",
       "      <td>[0.9494444444444443]</td>\n",
       "      <td>[0.9369444444444444]</td>\n",
       "      <td>[0.9369444444444444]</td>\n",
       "      <td>[0.9494444444444443]</td>\n",
       "      <td>[0.9494444444444443]</td>\n",
       "      <td>[0.8897222222222222]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9494444444444443]</td>\n",
       "      <td>[0.9369444444444444]</td>\n",
       "      <td>[0.9369444444444444]</td>\n",
       "      <td>[0.9494444444444443]</td>\n",
       "      <td>[0.9494444444444443]</td>\n",
       "      <td>[0.9605555555555554]</td>\n",
       "      <td>[0.8699999999999999]</td>\n",
       "      <td>[0.9480555555555554]</td>\n",
       "      <td>[0.9605555555555554]</td>\n",
       "      <td>[0.9605555555555554]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AUC_macro</th>\n",
       "      <td>[0.9969444444444445]</td>\n",
       "      <td>[0.9986111111111111]</td>\n",
       "      <td>[0.9986111111111111]</td>\n",
       "      <td>[0.9969444444444445]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9986111111111111]</td>\n",
       "      <td>[0.9986111111111111]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9944444444444442]</td>\n",
       "      <td>...</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9969444444444445]</td>\n",
       "      <td>[0.9986111111111111]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9966666666666667]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9966666666666667]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9983333333333334]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision_score_weighted</th>\n",
       "      <td>[0.9726747109100049]</td>\n",
       "      <td>[0.9781045751633988]</td>\n",
       "      <td>[0.9726747109100049]</td>\n",
       "      <td>[0.9792106586224232]</td>\n",
       "      <td>[0.9792106586224232]</td>\n",
       "      <td>[0.9726747109100049]</td>\n",
       "      <td>[0.9726747109100049]</td>\n",
       "      <td>[0.9792106586224232]</td>\n",
       "      <td>[0.9792106586224232]</td>\n",
       "      <td>[0.9509286791639733]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9792106586224232]</td>\n",
       "      <td>[0.9726747109100049]</td>\n",
       "      <td>[0.9726747109100049]</td>\n",
       "      <td>[0.9792106586224232]</td>\n",
       "      <td>[0.9792106586224232]</td>\n",
       "      <td>[0.9844394167923578]</td>\n",
       "      <td>[0.9487516568398922]</td>\n",
       "      <td>[0.9791452991452992]</td>\n",
       "      <td>[0.9844394167923578]</td>\n",
       "      <td>[0.9844394167923578]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall_score_micro</th>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9470588235294117]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "      <td>[0.9352941176470588]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>matthews_correlation</th>\n",
       "      <td>[0.9386123000100224]</td>\n",
       "      <td>[0.9526785151252903]</td>\n",
       "      <td>[0.9386123000100224]</td>\n",
       "      <td>[0.9511123000100223]</td>\n",
       "      <td>[0.9511123000100223]</td>\n",
       "      <td>[0.9386123000100224]</td>\n",
       "      <td>[0.9386123000100224]</td>\n",
       "      <td>[0.9511123000100223]</td>\n",
       "      <td>[0.9511123000100223]</td>\n",
       "      <td>[0.8933996344249813]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9511123000100223]</td>\n",
       "      <td>[0.9386123000100224]</td>\n",
       "      <td>[0.9386123000100224]</td>\n",
       "      <td>[0.9511123000100223]</td>\n",
       "      <td>[0.9511123000100223]</td>\n",
       "      <td>[0.9622234111211334]</td>\n",
       "      <td>[0.879046503881599]</td>\n",
       "      <td>[0.9509646078676276]</td>\n",
       "      <td>[0.9622234111211334]</td>\n",
       "      <td>[0.9622234111211334]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision_score_micro</th>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9470588235294117]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "      <td>[0.9352941176470588]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>average_precision_score_micro</th>\n",
       "      <td>[0.9958391762277665]</td>\n",
       "      <td>[0.9974351969890124]</td>\n",
       "      <td>[0.996477594649831]</td>\n",
       "      <td>[0.9961467502377627]</td>\n",
       "      <td>[0.9986928104575163]</td>\n",
       "      <td>[0.996477594649831]</td>\n",
       "      <td>[0.9971083996033914]</td>\n",
       "      <td>[0.9986928104575163]</td>\n",
       "      <td>[0.9986928104575163]</td>\n",
       "      <td>[0.9922659012005214]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9986928104575163]</td>\n",
       "      <td>[0.9971083996033914]</td>\n",
       "      <td>[0.9977468180254558]</td>\n",
       "      <td>[0.9986928104575163]</td>\n",
       "      <td>[0.9986928104575163]</td>\n",
       "      <td>[0.997111966045448]</td>\n",
       "      <td>[0.9852994134774894]</td>\n",
       "      <td>[0.9974235870818916]</td>\n",
       "      <td>[0.9983811894210728]</td>\n",
       "      <td>[0.9983811894210728]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9470588235294117]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "      <td>[0.9352941176470588]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>log_loss</th>\n",
       "      <td>[0.06953465307505337]</td>\n",
       "      <td>[0.283771576708101]</td>\n",
       "      <td>[0.14099732417688277]</td>\n",
       "      <td>[0.09169236604145156]</td>\n",
       "      <td>[0.07861864365502563]</td>\n",
       "      <td>[0.12424090632583987]</td>\n",
       "      <td>[0.14233186299666456]</td>\n",
       "      <td>[0.08276036113345764]</td>\n",
       "      <td>[0.041735018418679284]</td>\n",
       "      <td>[0.32475337134579546]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.048804713895068776]</td>\n",
       "      <td>[0.2590568021043464]</td>\n",
       "      <td>[0.27928275608402325]</td>\n",
       "      <td>[0.061556207339617766]</td>\n",
       "      <td>[0.05428317131764757]</td>\n",
       "      <td>[0.09697331347511076]</td>\n",
       "      <td>[0.17850882380610023]</td>\n",
       "      <td>[0.14435074357818634]</td>\n",
       "      <td>[0.2339590306444852]</td>\n",
       "      <td>[0.06470877571022829]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted_accuracy</th>\n",
       "      <td>[0.9720097939196082]</td>\n",
       "      <td>[0.9749683738012651]</td>\n",
       "      <td>[0.9720097939196082]</td>\n",
       "      <td>[0.9775270352989185]</td>\n",
       "      <td>[0.9775270352989185]</td>\n",
       "      <td>[0.9720097939196082]</td>\n",
       "      <td>[0.9720097939196082]</td>\n",
       "      <td>[0.9775270352989185]</td>\n",
       "      <td>[0.9775270352989185]</td>\n",
       "      <td>[0.9485615180575392]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9775270352989185]</td>\n",
       "      <td>[0.9720097939196082]</td>\n",
       "      <td>[0.9720097939196082]</td>\n",
       "      <td>[0.9775270352989185]</td>\n",
       "      <td>[0.9775270352989185]</td>\n",
       "      <td>[0.9837339318506426]</td>\n",
       "      <td>[0.9349535569374282]</td>\n",
       "      <td>[0.9782166904713323]</td>\n",
       "      <td>[0.9837339318506426]</td>\n",
       "      <td>[0.9837339318506426]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall_score_macro</th>\n",
       "      <td>[0.9684722222222222]</td>\n",
       "      <td>[0.9784722222222223]</td>\n",
       "      <td>[0.9684722222222222]</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9684722222222222]</td>\n",
       "      <td>[0.9684722222222222]</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9448611111111112]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9684722222222222]</td>\n",
       "      <td>[0.9684722222222222]</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9747222222222222]</td>\n",
       "      <td>[0.9802777777777777]</td>\n",
       "      <td>[0.9349999999999999]</td>\n",
       "      <td>[0.9740277777777777]</td>\n",
       "      <td>[0.9802777777777777]</td>\n",
       "      <td>[0.9802777777777777]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>average_precision_score_weighted</th>\n",
       "      <td>[0.997325289089995]</td>\n",
       "      <td>[0.9987581699346405]</td>\n",
       "      <td>[0.9987581699346405]</td>\n",
       "      <td>[0.997325289089995]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9987581699346405]</td>\n",
       "      <td>[0.9987581699346405]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9951767676767677]</td>\n",
       "      <td>...</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.997325289089995]</td>\n",
       "      <td>[0.9987581699346405]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9973766429648784]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9973766429648784]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9985671191553545]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision_score_macro</th>\n",
       "      <td>[0.9704594017094017]</td>\n",
       "      <td>[0.9743055555555555]</td>\n",
       "      <td>[0.9704594017094017]</td>\n",
       "      <td>[0.9767094017094017]</td>\n",
       "      <td>[0.9767094017094017]</td>\n",
       "      <td>[0.9704594017094017]</td>\n",
       "      <td>[0.9704594017094017]</td>\n",
       "      <td>[0.9767094017094017]</td>\n",
       "      <td>[0.9767094017094017]</td>\n",
       "      <td>[0.9488721001221002]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9767094017094017]</td>\n",
       "      <td>[0.9704594017094017]</td>\n",
       "      <td>[0.9704594017094017]</td>\n",
       "      <td>[0.9767094017094017]</td>\n",
       "      <td>[0.9767094017094017]</td>\n",
       "      <td>[0.9822649572649572]</td>\n",
       "      <td>[0.9447018259518261]</td>\n",
       "      <td>[0.9772649572649573]</td>\n",
       "      <td>[0.9822649572649572]</td>\n",
       "      <td>[0.9822649572649572]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall_score_weighted</th>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9470588235294117]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9705882352941178]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "      <td>[0.9352941176470588]</td>\n",
       "      <td>[0.9764705882352942]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "      <td>[0.9823529411764707]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1_score_macro</th>\n",
       "      <td>[0.9680371683587795]</td>\n",
       "      <td>[0.9755927239143352]</td>\n",
       "      <td>[0.9680371683587795]</td>\n",
       "      <td>[0.9739603709731586]</td>\n",
       "      <td>[0.9739603709731586]</td>\n",
       "      <td>[0.9680371683587795]</td>\n",
       "      <td>[0.9680371683587795]</td>\n",
       "      <td>[0.9739603709731586]</td>\n",
       "      <td>[0.9739603709731586]</td>\n",
       "      <td>[0.9443434979081433]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9739603709731586]</td>\n",
       "      <td>[0.9680371683587795]</td>\n",
       "      <td>[0.9680371683587795]</td>\n",
       "      <td>[0.9739603709731586]</td>\n",
       "      <td>[0.9739603709731586]</td>\n",
       "      <td>[0.979842723914335]</td>\n",
       "      <td>[0.9315668872638856]</td>\n",
       "      <td>[0.9738778116336334]</td>\n",
       "      <td>[0.979842723914335]</td>\n",
       "      <td>[0.979842723914335]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AUC_micro</th>\n",
       "      <td>[0.9955017301038062]</td>\n",
       "      <td>[0.9972318339100348]</td>\n",
       "      <td>[0.9961937716262976]</td>\n",
       "      <td>[0.9958477508650517]</td>\n",
       "      <td>[0.9986159169550172]</td>\n",
       "      <td>[0.9961937716262976]</td>\n",
       "      <td>[0.996885813148789]</td>\n",
       "      <td>[0.9986159169550172]</td>\n",
       "      <td>[0.9986159169550172]</td>\n",
       "      <td>[0.9913494809688583]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9986159169550172]</td>\n",
       "      <td>[0.996885813148789]</td>\n",
       "      <td>[0.9975778546712804]</td>\n",
       "      <td>[0.9986159169550172]</td>\n",
       "      <td>[0.9986159169550172]</td>\n",
       "      <td>[0.9968858131487888]</td>\n",
       "      <td>[0.9837370242214533]</td>\n",
       "      <td>[0.9972318339100346]</td>\n",
       "      <td>[0.9982698961937716]</td>\n",
       "      <td>[0.9982698961937716]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1_score_weighted</th>\n",
       "      <td>[0.970518587408613]</td>\n",
       "      <td>[0.9766100906765869]</td>\n",
       "      <td>[0.970518587408613]</td>\n",
       "      <td>[0.9764009403497894]</td>\n",
       "      <td>[0.9764009403497894]</td>\n",
       "      <td>[0.970518587408613]</td>\n",
       "      <td>[0.970518587408613]</td>\n",
       "      <td>[0.9764009403497894]</td>\n",
       "      <td>[0.9764009403497894]</td>\n",
       "      <td>[0.9467844972813341]</td>\n",
       "      <td>...</td>\n",
       "      <td>[0.9764009403497894]</td>\n",
       "      <td>[0.970518587408613]</td>\n",
       "      <td>[0.970518587408613]</td>\n",
       "      <td>[0.9764009403497894]</td>\n",
       "      <td>[0.9764009403497894]</td>\n",
       "      <td>[0.9822832932909659]</td>\n",
       "      <td>[0.9340052452296362]</td>\n",
       "      <td>[0.9763596606800269]</td>\n",
       "      <td>[0.9822832932909659]</td>\n",
       "      <td>[0.9822832932909659]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>AUC_weighted</th>\n",
       "      <td>[0.9969444444444445]</td>\n",
       "      <td>[0.9986111111111111]</td>\n",
       "      <td>[0.9986111111111111]</td>\n",
       "      <td>[0.9969444444444445]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9986111111111111]</td>\n",
       "      <td>[0.9986111111111111]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9944444444444445]</td>\n",
       "      <td>...</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9969444444444445]</td>\n",
       "      <td>[0.9986111111111111]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9966666666666667]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9966666666666667]</td>\n",
       "      <td>[1.0]</td>\n",
       "      <td>[0.9983333333333334]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_1  \\\n",
       "average_precision_score_macro                      [0.9967628205128205]   \n",
       "f1_score_micro                                     [0.9705882352941178]   \n",
       "balanced_accuracy                                  [0.9684722222222222]   \n",
       "norm_macro_recall                                  [0.9369444444444444]   \n",
       "AUC_macro                                          [0.9969444444444445]   \n",
       "precision_score_weighted                           [0.9726747109100049]   \n",
       "recall_score_micro                                 [0.9705882352941178]   \n",
       "matthews_correlation                               [0.9386123000100224]   \n",
       "precision_score_micro                              [0.9705882352941178]   \n",
       "average_precision_score_micro                      [0.9958391762277665]   \n",
       "accuracy                                           [0.9705882352941178]   \n",
       "log_loss                                          [0.06953465307505337]   \n",
       "weighted_accuracy                                  [0.9720097939196082]   \n",
       "recall_score_macro                                 [0.9684722222222222]   \n",
       "average_precision_score_weighted                    [0.997325289089995]   \n",
       "precision_score_macro                              [0.9704594017094017]   \n",
       "recall_score_weighted                              [0.9705882352941178]   \n",
       "f1_score_macro                                     [0.9680371683587795]   \n",
       "AUC_micro                                          [0.9955017301038062]   \n",
       "f1_score_weighted                                   [0.970518587408613]   \n",
       "AUC_weighted                                       [0.9969444444444445]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_3  \\\n",
       "average_precision_score_macro                                 [0.99875]   \n",
       "f1_score_micro                                     [0.9764705882352942]   \n",
       "balanced_accuracy                                  [0.9784722222222223]   \n",
       "norm_macro_recall                                  [0.9569444444444445]   \n",
       "AUC_macro                                          [0.9986111111111111]   \n",
       "precision_score_weighted                           [0.9781045751633988]   \n",
       "recall_score_micro                                 [0.9764705882352942]   \n",
       "matthews_correlation                               [0.9526785151252903]   \n",
       "precision_score_micro                              [0.9764705882352942]   \n",
       "average_precision_score_micro                      [0.9974351969890124]   \n",
       "accuracy                                           [0.9764705882352942]   \n",
       "log_loss                                            [0.283771576708101]   \n",
       "weighted_accuracy                                  [0.9749683738012651]   \n",
       "recall_score_macro                                 [0.9784722222222223]   \n",
       "average_precision_score_weighted                   [0.9987581699346405]   \n",
       "precision_score_macro                              [0.9743055555555555]   \n",
       "recall_score_weighted                              [0.9764705882352942]   \n",
       "f1_score_macro                                     [0.9755927239143352]   \n",
       "AUC_micro                                          [0.9972318339100348]   \n",
       "f1_score_weighted                                  [0.9766100906765869]   \n",
       "AUC_weighted                                       [0.9986111111111111]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_4  \\\n",
       "average_precision_score_macro                                 [0.99875]   \n",
       "f1_score_micro                                     [0.9705882352941178]   \n",
       "balanced_accuracy                                  [0.9684722222222222]   \n",
       "norm_macro_recall                                  [0.9369444444444444]   \n",
       "AUC_macro                                          [0.9986111111111111]   \n",
       "precision_score_weighted                           [0.9726747109100049]   \n",
       "recall_score_micro                                 [0.9705882352941178]   \n",
       "matthews_correlation                               [0.9386123000100224]   \n",
       "precision_score_micro                              [0.9705882352941178]   \n",
       "average_precision_score_micro                       [0.996477594649831]   \n",
       "accuracy                                           [0.9705882352941178]   \n",
       "log_loss                                          [0.14099732417688277]   \n",
       "weighted_accuracy                                  [0.9720097939196082]   \n",
       "recall_score_macro                                 [0.9684722222222222]   \n",
       "average_precision_score_weighted                   [0.9987581699346405]   \n",
       "precision_score_macro                              [0.9704594017094017]   \n",
       "recall_score_weighted                              [0.9705882352941178]   \n",
       "f1_score_macro                                     [0.9680371683587795]   \n",
       "AUC_micro                                          [0.9961937716262976]   \n",
       "f1_score_weighted                                   [0.970518587408613]   \n",
       "AUC_weighted                                       [0.9986111111111111]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_0  \\\n",
       "average_precision_score_macro                      [0.9967628205128205]   \n",
       "f1_score_micro                                     [0.9764705882352942]   \n",
       "balanced_accuracy                                  [0.9747222222222222]   \n",
       "norm_macro_recall                                  [0.9494444444444443]   \n",
       "AUC_macro                                          [0.9969444444444445]   \n",
       "precision_score_weighted                           [0.9792106586224232]   \n",
       "recall_score_micro                                 [0.9764705882352942]   \n",
       "matthews_correlation                               [0.9511123000100223]   \n",
       "precision_score_micro                              [0.9764705882352942]   \n",
       "average_precision_score_micro                      [0.9961467502377627]   \n",
       "accuracy                                           [0.9764705882352942]   \n",
       "log_loss                                          [0.09169236604145156]   \n",
       "weighted_accuracy                                  [0.9775270352989185]   \n",
       "recall_score_macro                                 [0.9747222222222222]   \n",
       "average_precision_score_weighted                    [0.997325289089995]   \n",
       "precision_score_macro                              [0.9767094017094017]   \n",
       "recall_score_weighted                              [0.9764705882352942]   \n",
       "f1_score_macro                                     [0.9739603709731586]   \n",
       "AUC_micro                                          [0.9958477508650517]   \n",
       "f1_score_weighted                                  [0.9764009403497894]   \n",
       "AUC_weighted                                       [0.9969444444444445]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_15  \\\n",
       "average_precision_score_macro                                      [1.0]   \n",
       "f1_score_micro                                      [0.9764705882352942]   \n",
       "balanced_accuracy                                   [0.9747222222222222]   \n",
       "norm_macro_recall                                   [0.9494444444444443]   \n",
       "AUC_macro                                                          [1.0]   \n",
       "precision_score_weighted                            [0.9792106586224232]   \n",
       "recall_score_micro                                  [0.9764705882352942]   \n",
       "matthews_correlation                                [0.9511123000100223]   \n",
       "precision_score_micro                               [0.9764705882352942]   \n",
       "average_precision_score_micro                       [0.9986928104575163]   \n",
       "accuracy                                            [0.9764705882352942]   \n",
       "log_loss                                           [0.07861864365502563]   \n",
       "weighted_accuracy                                   [0.9775270352989185]   \n",
       "recall_score_macro                                  [0.9747222222222222]   \n",
       "average_precision_score_weighted                                   [1.0]   \n",
       "precision_score_macro                               [0.9767094017094017]   \n",
       "recall_score_weighted                               [0.9764705882352942]   \n",
       "f1_score_macro                                      [0.9739603709731586]   \n",
       "AUC_micro                                           [0.9986159169550172]   \n",
       "f1_score_weighted                                   [0.9764009403497894]   \n",
       "AUC_weighted                                                       [1.0]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_18  \\\n",
       "average_precision_score_macro                                  [0.99875]   \n",
       "f1_score_micro                                      [0.9705882352941178]   \n",
       "balanced_accuracy                                   [0.9684722222222222]   \n",
       "norm_macro_recall                                   [0.9369444444444444]   \n",
       "AUC_macro                                           [0.9986111111111111]   \n",
       "precision_score_weighted                            [0.9726747109100049]   \n",
       "recall_score_micro                                  [0.9705882352941178]   \n",
       "matthews_correlation                                [0.9386123000100224]   \n",
       "precision_score_micro                               [0.9705882352941178]   \n",
       "average_precision_score_micro                        [0.996477594649831]   \n",
       "accuracy                                            [0.9705882352941178]   \n",
       "log_loss                                           [0.12424090632583987]   \n",
       "weighted_accuracy                                   [0.9720097939196082]   \n",
       "recall_score_macro                                  [0.9684722222222222]   \n",
       "average_precision_score_weighted                    [0.9987581699346405]   \n",
       "precision_score_macro                               [0.9704594017094017]   \n",
       "recall_score_weighted                               [0.9705882352941178]   \n",
       "f1_score_macro                                      [0.9680371683587795]   \n",
       "AUC_micro                                           [0.9961937716262976]   \n",
       "f1_score_weighted                                    [0.970518587408613]   \n",
       "AUC_weighted                                        [0.9986111111111111]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_10  \\\n",
       "average_precision_score_macro                                  [0.99875]   \n",
       "f1_score_micro                                      [0.9705882352941178]   \n",
       "balanced_accuracy                                   [0.9684722222222222]   \n",
       "norm_macro_recall                                   [0.9369444444444444]   \n",
       "AUC_macro                                           [0.9986111111111111]   \n",
       "precision_score_weighted                            [0.9726747109100049]   \n",
       "recall_score_micro                                  [0.9705882352941178]   \n",
       "matthews_correlation                                [0.9386123000100224]   \n",
       "precision_score_micro                               [0.9705882352941178]   \n",
       "average_precision_score_micro                       [0.9971083996033914]   \n",
       "accuracy                                            [0.9705882352941178]   \n",
       "log_loss                                           [0.14233186299666456]   \n",
       "weighted_accuracy                                   [0.9720097939196082]   \n",
       "recall_score_macro                                  [0.9684722222222222]   \n",
       "average_precision_score_weighted                    [0.9987581699346405]   \n",
       "precision_score_macro                               [0.9704594017094017]   \n",
       "recall_score_weighted                               [0.9705882352941178]   \n",
       "f1_score_macro                                      [0.9680371683587795]   \n",
       "AUC_micro                                            [0.996885813148789]   \n",
       "f1_score_weighted                                    [0.970518587408613]   \n",
       "AUC_weighted                                        [0.9986111111111111]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_12  \\\n",
       "average_precision_score_macro                                      [1.0]   \n",
       "f1_score_micro                                      [0.9764705882352942]   \n",
       "balanced_accuracy                                   [0.9747222222222222]   \n",
       "norm_macro_recall                                   [0.9494444444444443]   \n",
       "AUC_macro                                                          [1.0]   \n",
       "precision_score_weighted                            [0.9792106586224232]   \n",
       "recall_score_micro                                  [0.9764705882352942]   \n",
       "matthews_correlation                                [0.9511123000100223]   \n",
       "precision_score_micro                               [0.9764705882352942]   \n",
       "average_precision_score_micro                       [0.9986928104575163]   \n",
       "accuracy                                            [0.9764705882352942]   \n",
       "log_loss                                           [0.08276036113345764]   \n",
       "weighted_accuracy                                   [0.9775270352989185]   \n",
       "recall_score_macro                                  [0.9747222222222222]   \n",
       "average_precision_score_weighted                                   [1.0]   \n",
       "precision_score_macro                               [0.9767094017094017]   \n",
       "recall_score_weighted                               [0.9764705882352942]   \n",
       "f1_score_macro                                      [0.9739603709731586]   \n",
       "AUC_micro                                           [0.9986159169550172]   \n",
       "f1_score_weighted                                   [0.9764009403497894]   \n",
       "AUC_weighted                                                       [1.0]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_13  \\\n",
       "average_precision_score_macro                                      [1.0]   \n",
       "f1_score_micro                                      [0.9764705882352942]   \n",
       "balanced_accuracy                                   [0.9747222222222222]   \n",
       "norm_macro_recall                                   [0.9494444444444443]   \n",
       "AUC_macro                                                          [1.0]   \n",
       "precision_score_weighted                            [0.9792106586224232]   \n",
       "recall_score_micro                                  [0.9764705882352942]   \n",
       "matthews_correlation                                [0.9511123000100223]   \n",
       "precision_score_micro                               [0.9764705882352942]   \n",
       "average_precision_score_micro                       [0.9986928104575163]   \n",
       "accuracy                                            [0.9764705882352942]   \n",
       "log_loss                                          [0.041735018418679284]   \n",
       "weighted_accuracy                                   [0.9775270352989185]   \n",
       "recall_score_macro                                  [0.9747222222222222]   \n",
       "average_precision_score_weighted                                   [1.0]   \n",
       "precision_score_macro                               [0.9767094017094017]   \n",
       "recall_score_weighted                               [0.9764705882352942]   \n",
       "f1_score_macro                                      [0.9739603709731586]   \n",
       "AUC_micro                                           [0.9986159169550172]   \n",
       "f1_score_weighted                                   [0.9764009403497894]   \n",
       "AUC_weighted                                                       [1.0]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_14  ...  \\\n",
       "average_precision_score_macro                       [0.9951781705948374]  ...   \n",
       "f1_score_micro                                      [0.9470588235294117]  ...   \n",
       "balanced_accuracy                                   [0.9448611111111112]  ...   \n",
       "norm_macro_recall                                   [0.8897222222222222]  ...   \n",
       "AUC_macro                                           [0.9944444444444442]  ...   \n",
       "precision_score_weighted                            [0.9509286791639733]  ...   \n",
       "recall_score_micro                                  [0.9470588235294117]  ...   \n",
       "matthews_correlation                                [0.8933996344249813]  ...   \n",
       "precision_score_micro                               [0.9470588235294117]  ...   \n",
       "average_precision_score_micro                       [0.9922659012005214]  ...   \n",
       "accuracy                                            [0.9470588235294117]  ...   \n",
       "log_loss                                           [0.32475337134579546]  ...   \n",
       "weighted_accuracy                                   [0.9485615180575392]  ...   \n",
       "recall_score_macro                                  [0.9448611111111112]  ...   \n",
       "average_precision_score_weighted                    [0.9951767676767677]  ...   \n",
       "precision_score_macro                               [0.9488721001221002]  ...   \n",
       "recall_score_weighted                               [0.9470588235294117]  ...   \n",
       "f1_score_macro                                      [0.9443434979081433]  ...   \n",
       "AUC_micro                                           [0.9913494809688583]  ...   \n",
       "f1_score_weighted                                   [0.9467844972813341]  ...   \n",
       "AUC_weighted                                        [0.9944444444444445]  ...   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_7  \\\n",
       "average_precision_score_macro                                     [1.0]   \n",
       "f1_score_micro                                     [0.9764705882352942]   \n",
       "balanced_accuracy                                  [0.9747222222222222]   \n",
       "norm_macro_recall                                  [0.9494444444444443]   \n",
       "AUC_macro                                                         [1.0]   \n",
       "precision_score_weighted                           [0.9792106586224232]   \n",
       "recall_score_micro                                 [0.9764705882352942]   \n",
       "matthews_correlation                               [0.9511123000100223]   \n",
       "precision_score_micro                              [0.9764705882352942]   \n",
       "average_precision_score_micro                      [0.9986928104575163]   \n",
       "accuracy                                           [0.9764705882352942]   \n",
       "log_loss                                         [0.048804713895068776]   \n",
       "weighted_accuracy                                  [0.9775270352989185]   \n",
       "recall_score_macro                                 [0.9747222222222222]   \n",
       "average_precision_score_weighted                                  [1.0]   \n",
       "precision_score_macro                              [0.9767094017094017]   \n",
       "recall_score_weighted                              [0.9764705882352942]   \n",
       "f1_score_macro                                     [0.9739603709731586]   \n",
       "AUC_micro                                          [0.9986159169550172]   \n",
       "f1_score_weighted                                  [0.9764009403497894]   \n",
       "AUC_weighted                                                      [1.0]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_6  \\\n",
       "average_precision_score_macro                      [0.9967628205128205]   \n",
       "f1_score_micro                                     [0.9705882352941178]   \n",
       "balanced_accuracy                                  [0.9684722222222222]   \n",
       "norm_macro_recall                                  [0.9369444444444444]   \n",
       "AUC_macro                                          [0.9969444444444445]   \n",
       "precision_score_weighted                           [0.9726747109100049]   \n",
       "recall_score_micro                                 [0.9705882352941178]   \n",
       "matthews_correlation                               [0.9386123000100224]   \n",
       "precision_score_micro                              [0.9705882352941178]   \n",
       "average_precision_score_micro                      [0.9971083996033914]   \n",
       "accuracy                                           [0.9705882352941178]   \n",
       "log_loss                                           [0.2590568021043464]   \n",
       "weighted_accuracy                                  [0.9720097939196082]   \n",
       "recall_score_macro                                 [0.9684722222222222]   \n",
       "average_precision_score_weighted                    [0.997325289089995]   \n",
       "precision_score_macro                              [0.9704594017094017]   \n",
       "recall_score_weighted                              [0.9705882352941178]   \n",
       "f1_score_macro                                     [0.9680371683587795]   \n",
       "AUC_micro                                           [0.996885813148789]   \n",
       "f1_score_weighted                                   [0.970518587408613]   \n",
       "AUC_weighted                                       [0.9969444444444445]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_11  \\\n",
       "average_precision_score_macro                                  [0.99875]   \n",
       "f1_score_micro                                      [0.9705882352941178]   \n",
       "balanced_accuracy                                   [0.9684722222222222]   \n",
       "norm_macro_recall                                   [0.9369444444444444]   \n",
       "AUC_macro                                           [0.9986111111111111]   \n",
       "precision_score_weighted                            [0.9726747109100049]   \n",
       "recall_score_micro                                  [0.9705882352941178]   \n",
       "matthews_correlation                                [0.9386123000100224]   \n",
       "precision_score_micro                               [0.9705882352941178]   \n",
       "average_precision_score_micro                       [0.9977468180254558]   \n",
       "accuracy                                            [0.9705882352941178]   \n",
       "log_loss                                           [0.27928275608402325]   \n",
       "weighted_accuracy                                   [0.9720097939196082]   \n",
       "recall_score_macro                                  [0.9684722222222222]   \n",
       "average_precision_score_weighted                    [0.9987581699346405]   \n",
       "precision_score_macro                               [0.9704594017094017]   \n",
       "recall_score_weighted                               [0.9705882352941178]   \n",
       "f1_score_macro                                      [0.9680371683587795]   \n",
       "AUC_micro                                           [0.9975778546712804]   \n",
       "f1_score_weighted                                    [0.970518587408613]   \n",
       "AUC_weighted                                        [0.9986111111111111]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_16  \\\n",
       "average_precision_score_macro                                      [1.0]   \n",
       "f1_score_micro                                      [0.9764705882352942]   \n",
       "balanced_accuracy                                   [0.9747222222222222]   \n",
       "norm_macro_recall                                   [0.9494444444444443]   \n",
       "AUC_macro                                                          [1.0]   \n",
       "precision_score_weighted                            [0.9792106586224232]   \n",
       "recall_score_micro                                  [0.9764705882352942]   \n",
       "matthews_correlation                                [0.9511123000100223]   \n",
       "precision_score_micro                               [0.9764705882352942]   \n",
       "average_precision_score_micro                       [0.9986928104575163]   \n",
       "accuracy                                            [0.9764705882352942]   \n",
       "log_loss                                          [0.061556207339617766]   \n",
       "weighted_accuracy                                   [0.9775270352989185]   \n",
       "recall_score_macro                                  [0.9747222222222222]   \n",
       "average_precision_score_weighted                                   [1.0]   \n",
       "precision_score_macro                               [0.9767094017094017]   \n",
       "recall_score_weighted                               [0.9764705882352942]   \n",
       "f1_score_macro                                      [0.9739603709731586]   \n",
       "AUC_micro                                           [0.9986159169550172]   \n",
       "f1_score_weighted                                   [0.9764009403497894]   \n",
       "AUC_weighted                                                       [1.0]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_33  \\\n",
       "average_precision_score_macro                                      [1.0]   \n",
       "f1_score_micro                                      [0.9764705882352942]   \n",
       "balanced_accuracy                                   [0.9747222222222222]   \n",
       "norm_macro_recall                                   [0.9494444444444443]   \n",
       "AUC_macro                                                          [1.0]   \n",
       "precision_score_weighted                            [0.9792106586224232]   \n",
       "recall_score_micro                                  [0.9764705882352942]   \n",
       "matthews_correlation                                [0.9511123000100223]   \n",
       "precision_score_micro                               [0.9764705882352942]   \n",
       "average_precision_score_micro                       [0.9986928104575163]   \n",
       "accuracy                                            [0.9764705882352942]   \n",
       "log_loss                                           [0.05428317131764757]   \n",
       "weighted_accuracy                                   [0.9775270352989185]   \n",
       "recall_score_macro                                  [0.9747222222222222]   \n",
       "average_precision_score_weighted                                   [1.0]   \n",
       "precision_score_macro                               [0.9767094017094017]   \n",
       "recall_score_weighted                               [0.9764705882352942]   \n",
       "f1_score_macro                                      [0.9739603709731586]   \n",
       "AUC_micro                                           [0.9986159169550172]   \n",
       "f1_score_weighted                                   [0.9764009403497894]   \n",
       "AUC_weighted                                                       [1.0]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_24  \\\n",
       "average_precision_score_macro                        [0.996475122100122]   \n",
       "f1_score_micro                                      [0.9823529411764707]   \n",
       "balanced_accuracy                                   [0.9802777777777777]   \n",
       "norm_macro_recall                                   [0.9605555555555554]   \n",
       "AUC_macro                                           [0.9966666666666667]   \n",
       "precision_score_weighted                            [0.9844394167923578]   \n",
       "recall_score_micro                                  [0.9823529411764707]   \n",
       "matthews_correlation                                [0.9622234111211334]   \n",
       "precision_score_micro                               [0.9823529411764707]   \n",
       "average_precision_score_micro                        [0.997111966045448]   \n",
       "accuracy                                            [0.9823529411764707]   \n",
       "log_loss                                           [0.09697331347511076]   \n",
       "weighted_accuracy                                   [0.9837339318506426]   \n",
       "recall_score_macro                                  [0.9802777777777777]   \n",
       "average_precision_score_weighted                    [0.9973766429648784]   \n",
       "precision_score_macro                               [0.9822649572649572]   \n",
       "recall_score_weighted                               [0.9823529411764707]   \n",
       "f1_score_macro                                       [0.979842723914335]   \n",
       "AUC_micro                                           [0.9968858131487888]   \n",
       "f1_score_weighted                                   [0.9822832932909659]   \n",
       "AUC_weighted                                        [0.9966666666666667]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_20  \\\n",
       "average_precision_score_macro                                      [1.0]   \n",
       "f1_score_micro                                      [0.9352941176470588]   \n",
       "balanced_accuracy                                   [0.9349999999999999]   \n",
       "norm_macro_recall                                   [0.8699999999999999]   \n",
       "AUC_macro                                                          [1.0]   \n",
       "precision_score_weighted                            [0.9487516568398922]   \n",
       "recall_score_micro                                  [0.9352941176470588]   \n",
       "matthews_correlation                                 [0.879046503881599]   \n",
       "precision_score_micro                               [0.9352941176470588]   \n",
       "average_precision_score_micro                       [0.9852994134774894]   \n",
       "accuracy                                            [0.9352941176470588]   \n",
       "log_loss                                           [0.17850882380610023]   \n",
       "weighted_accuracy                                   [0.9349535569374282]   \n",
       "recall_score_macro                                  [0.9349999999999999]   \n",
       "average_precision_score_weighted                                   [1.0]   \n",
       "precision_score_macro                               [0.9447018259518261]   \n",
       "recall_score_weighted                               [0.9352941176470588]   \n",
       "f1_score_macro                                      [0.9315668872638856]   \n",
       "AUC_micro                                           [0.9837370242214533]   \n",
       "f1_score_weighted                                   [0.9340052452296362]   \n",
       "AUC_weighted                                                       [1.0]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_30  \\\n",
       "average_precision_score_macro                        [0.996475122100122]   \n",
       "f1_score_micro                                      [0.9764705882352942]   \n",
       "balanced_accuracy                                   [0.9740277777777777]   \n",
       "norm_macro_recall                                   [0.9480555555555554]   \n",
       "AUC_macro                                           [0.9966666666666667]   \n",
       "precision_score_weighted                            [0.9791452991452992]   \n",
       "recall_score_micro                                  [0.9764705882352942]   \n",
       "matthews_correlation                                [0.9509646078676276]   \n",
       "precision_score_micro                               [0.9764705882352942]   \n",
       "average_precision_score_micro                       [0.9974235870818916]   \n",
       "accuracy                                            [0.9764705882352942]   \n",
       "log_loss                                           [0.14435074357818634]   \n",
       "weighted_accuracy                                   [0.9782166904713323]   \n",
       "recall_score_macro                                  [0.9740277777777777]   \n",
       "average_precision_score_weighted                    [0.9973766429648784]   \n",
       "precision_score_macro                               [0.9772649572649573]   \n",
       "recall_score_weighted                               [0.9764705882352942]   \n",
       "f1_score_macro                                      [0.9738778116336334]   \n",
       "AUC_micro                                           [0.9972318339100346]   \n",
       "f1_score_weighted                                   [0.9763596606800269]   \n",
       "AUC_weighted                                        [0.9966666666666667]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_39  \\\n",
       "average_precision_score_macro                                      [1.0]   \n",
       "f1_score_micro                                      [0.9823529411764707]   \n",
       "balanced_accuracy                                   [0.9802777777777777]   \n",
       "norm_macro_recall                                   [0.9605555555555554]   \n",
       "AUC_macro                                                          [1.0]   \n",
       "precision_score_weighted                            [0.9844394167923578]   \n",
       "recall_score_micro                                  [0.9823529411764707]   \n",
       "matthews_correlation                                [0.9622234111211334]   \n",
       "precision_score_micro                               [0.9823529411764707]   \n",
       "average_precision_score_micro                       [0.9983811894210728]   \n",
       "accuracy                                            [0.9823529411764707]   \n",
       "log_loss                                            [0.2339590306444852]   \n",
       "weighted_accuracy                                   [0.9837339318506426]   \n",
       "recall_score_macro                                  [0.9802777777777777]   \n",
       "average_precision_score_weighted                                   [1.0]   \n",
       "precision_score_macro                               [0.9822649572649572]   \n",
       "recall_score_weighted                               [0.9823529411764707]   \n",
       "f1_score_macro                                       [0.979842723914335]   \n",
       "AUC_micro                                           [0.9982698961937716]   \n",
       "f1_score_weighted                                   [0.9822832932909659]   \n",
       "AUC_weighted                                                       [1.0]   \n",
       "\n",
       "                                 296c76e2-158b-4482-99e3-b14ef348bc65_27  \n",
       "average_precision_score_macro                       [0.9980128205128205]  \n",
       "f1_score_micro                                      [0.9823529411764707]  \n",
       "balanced_accuracy                                   [0.9802777777777777]  \n",
       "norm_macro_recall                                   [0.9605555555555554]  \n",
       "AUC_macro                                           [0.9983333333333334]  \n",
       "precision_score_weighted                            [0.9844394167923578]  \n",
       "recall_score_micro                                  [0.9823529411764707]  \n",
       "matthews_correlation                                [0.9622234111211334]  \n",
       "precision_score_micro                               [0.9823529411764707]  \n",
       "average_precision_score_micro                       [0.9983811894210728]  \n",
       "accuracy                                            [0.9823529411764707]  \n",
       "log_loss                                           [0.06470877571022829]  \n",
       "weighted_accuracy                                   [0.9837339318506426]  \n",
       "recall_score_macro                                  [0.9802777777777777]  \n",
       "average_precision_score_weighted                    [0.9985671191553545]  \n",
       "precision_score_macro                               [0.9822649572649572]  \n",
       "recall_score_weighted                               [0.9823529411764707]  \n",
       "f1_score_macro                                       [0.979842723914335]  \n",
       "AUC_micro                                           [0.9982698961937716]  \n",
       "f1_score_weighted                                   [0.9822832932909659]  \n",
       "AUC_weighted                                        [0.9983333333333334]  \n",
       "\n",
       "[21 rows x 37 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "with open(metrics_output._path_on_datastore) as f:\n",
    "    metrics_output_result = f.read()\n",
    "    \n",
    "deserialized_metrics_output = json.loads(metrics_output_result)\n",
    "df = pd.DataFrame(deserialized_metrics_output)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "gather": {
     "logged": 1640555608477
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading azureml/296c76e2-158b-4482-99e3-b14ef348bc65/model_data\n",
      "Downloaded azureml/296c76e2-158b-4482-99e3-b14ef348bc65/model_data, 1 files out of an estimated total of 1\n"
     ]
    }
   ],
   "source": [
    "# Retrieve best model from Pipeline Run\n",
    "best_model_output = pipeline_run.get_pipeline_output(best_model_output_name)\n",
    "num_file_downloaded = best_model_output.download('.', show_progress=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "gather": {
     "logged": 1640555635825
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('datatransformer',\n",
       "                 DataTransformer(enable_dnn=False, enable_feature_sweeping=True, feature_sweeping_config={}, feature_sweeping_timeout=86400, featurization_config=None, force_text_dnn=False, is_cross_validation=True, is_onnx_compatible=False, observer=None, task='classification', working_dir='/mnt/batch/tasks/shared/LS_root/mount...\n",
       "                 RandomForestClassifier(bootstrap=False, ccp_alpha=0.0,\n",
       "                                        class_weight=None, criterion='entropy',\n",
       "                                        max_depth=None, max_features=0.1,\n",
       "                                        max_leaf_nodes=None, max_samples=None,\n",
       "                                        min_impurity_decrease=0.0,\n",
       "                                        min_impurity_split=None,\n",
       "                                        min_samples_leaf=0.01,\n",
       "                                        min_samples_split=0.01,\n",
       "                                        min_weight_fraction_leaf=0.0,\n",
       "                                        n_estimators=50, n_jobs=1,\n",
       "                                        oob_score=False, random_state=None,\n",
       "                                        verbose=0, warm_start=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save the best model\n",
    "import pickle\n",
    "\n",
    "with open(best_model_output._path_on_datastore, \"rb\" ) as f:\n",
    "    best_model = pickle.load(f)\n",
    "best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "gather": {
     "logged": 1640555657364
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('datatransformer',\n",
       "  DataTransformer(\n",
       "      task='classification',\n",
       "      is_onnx_compatible=False,\n",
       "      enable_feature_sweeping=True,\n",
       "      enable_dnn=False,\n",
       "      force_text_dnn=False,\n",
       "      feature_sweeping_timeout=86400,\n",
       "      featurization_config=None,\n",
       "      is_cross_validation=True,\n",
       "      feature_sweeping_config={}\n",
       "  )),\n",
       " ('StandardScalerWrapper',\n",
       "  StandardScalerWrapper(\n",
       "      copy=True,\n",
       "      with_mean=False,\n",
       "      with_std=False\n",
       "  )),\n",
       " ('RandomForestClassifier',\n",
       "  RandomForestClassifier(bootstrap=False, ccp_alpha=0.0, class_weight=None,\n",
       "                         criterion='entropy', max_depth=None, max_features=0.1,\n",
       "                         max_leaf_nodes=None, max_samples=None,\n",
       "                         min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                         min_samples_leaf=0.01, min_samples_split=0.01,\n",
       "                         min_weight_fraction_leaf=0.0, n_estimators=50, n_jobs=1,\n",
       "                         oob_score=False, random_state=None, verbose=0,\n",
       "                         warm_start=False))]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "datatransformer\n",
      "{'enable_dnn': False,\n",
      " 'enable_feature_sweeping': True,\n",
      " 'feature_sweeping_config': {},\n",
      " 'feature_sweeping_timeout': 86400,\n",
      " 'featurization_config': None,\n",
      " 'force_text_dnn': False,\n",
      " 'is_cross_validation': True,\n",
      " 'is_onnx_compatible': False,\n",
      " 'observer': None,\n",
      " 'task': 'classification',\n",
      " 'working_dir': '/mnt/batch/tasks/shared/LS_root/mounts/clusters/notebook177093/code/divorce-prediction-azure-ml'}\n",
      "\n",
      "StandardScalerWrapper\n",
      "{'class_name': 'StandardScaler',\n",
      " 'copy': True,\n",
      " 'module_name': 'sklearn.preprocessing._data',\n",
      " 'with_mean': False,\n",
      " 'with_std': False}\n",
      "\n",
      "RandomForestClassifier\n",
      "{'bootstrap': False,\n",
      " 'ccp_alpha': 0.0,\n",
      " 'class_weight': None,\n",
      " 'criterion': 'entropy',\n",
      " 'max_depth': None,\n",
      " 'max_features': 0.1,\n",
      " 'max_leaf_nodes': None,\n",
      " 'max_samples': None,\n",
      " 'min_impurity_decrease': 0.0,\n",
      " 'min_impurity_split': None,\n",
      " 'min_samples_leaf': 0.01,\n",
      " 'min_samples_split': 0.01,\n",
      " 'min_weight_fraction_leaf': 0.0,\n",
      " 'n_estimators': 50,\n",
      " 'n_jobs': 1,\n",
      " 'oob_score': False,\n",
      " 'random_state': None,\n",
      " 'verbose': 0,\n",
      " 'warm_start': False}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "### Explore models and metrics\n",
    "from pprint import pprint\n",
    "\n",
    "def print_model(model, prefix=\"\"):\n",
    "    for step in model.steps:\n",
    "        print(prefix + step[0])\n",
    "        if hasattr(step[1], 'estimators') and hasattr(step[1], 'weights'):\n",
    "            pprint({'estimators': list(e[0] for e in step[1].estimators), 'weights': step[1].weights})\n",
    "            print()\n",
    "            for estimator in step[1].estimators:\n",
    "                print_model(estimator[1], estimator[0]+ ' - ')\n",
    "        elif hasattr(step[1], '_base_learners') and hasattr(step[1], '_meta_learner'):\n",
    "            print(\"\\nMeta Learner\")\n",
    "            pprint(step[1]._meta_learner)\n",
    "            print()\n",
    "            for estimator in step[1]._base_learners:\n",
    "                print_model(estimator[1], estimator[0]+ ' - ')\n",
    "        else:\n",
    "            pprint(step[1].get_params())\n",
    "            print()\n",
    "            \n",
    "print_model(best_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('datatransformer',\n",
       "                 DataTransformer(enable_dnn=False, enable_feature_sweeping=True, feature_sweeping_config={}, feature_sweeping_timeout=86400, featurization_config=None, force_text_dnn=False, is_cross_validation=True, is_onnx_compatible=False, observer=None, task='classification', working_dir='/mnt/batch/tasks/shared/LS_root/mount...\n",
       "                 RandomForestClassifier(bootstrap=False, ccp_alpha=0.0,\n",
       "                                        class_weight=None, criterion='entropy',\n",
       "                                        max_depth=None, max_features=0.1,\n",
       "                                        max_leaf_nodes=None, max_samples=None,\n",
       "                                        min_impurity_decrease=0.0,\n",
       "                                        min_impurity_split=None,\n",
       "                                        min_samples_leaf=0.01,\n",
       "                                        min_samples_split=0.01,\n",
       "                                        min_weight_fraction_leaf=0.0,\n",
       "                                        n_estimators=50, n_jobs=1,\n",
       "                                        oob_score=False, random_state=None,\n",
       "                                        verbose=0, warm_start=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:The version of the SDK does not match the version the model was trained on.\n",
      "WARNING:root:The consistency in the result may not be guaranteed.\n",
      "WARNING:root:Package:azureml-automl-core, training version:1.37.0, current version:1.36.0\n",
      "Package:azureml-automl-runtime, training version:1.37.0, current version:1.36.0\n",
      "Package:azureml-core, training version:1.37.0, current version:1.36.0.post2\n",
      "Package:azureml-dataprep, training version:2.25.0, current version:2.24.4\n",
      "Package:azureml-dataprep-rslex, training version:2.1.0, current version:2.0.3\n",
      "Package:azureml-dataset-runtime, training version:1.37.0, current version:1.36.0\n",
      "Package:azureml-defaults, training version:1.37.0, current version:1.36.0\n",
      "Package:azureml-interpret, training version:1.37.0, current version:1.36.0\n",
      "Package:azureml-mlflow, training version:1.37.0, current version:1.36.0\n",
      "Package:azureml-pipeline-core, training version:1.37.0, current version:1.36.0\n",
      "Package:azureml-responsibleai, training version:1.37.0, current version:1.36.0\n",
      "Package:azureml-telemetry, training version:1.37.0, current version:1.36.0\n",
      "Package:azureml-train-automl-client, training version:1.37.0, current version:1.36.0\n",
      "Package:azureml-train-automl-runtime, training version:1.37.0, current version:1.36.0\n",
      "Package:azureml-train-core, training version:1.37.0, current version:1.36.0\n",
      "Package:azureml-train-restclients-hyperdrive, training version:1.37.0, current version:1.36.0\n",
      "WARNING:root:Please ensure the version of your local conda dependencies match the version on which your model was trained in order to properly retrieve your model.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run(Experiment: ml-experiment-1,\n",
      "Id: 296c76e2-158b-4482-99e3-b14ef348bc65_17,\n",
      "Type: azureml.scriptrun,\n",
      "Status: Completed)\n",
      "datatransformer\n",
      "{'enable_dnn': False,\n",
      " 'enable_feature_sweeping': True,\n",
      " 'feature_sweeping_config': {},\n",
      " 'feature_sweeping_timeout': 86400,\n",
      " 'featurization_config': None,\n",
      " 'force_text_dnn': False,\n",
      " 'is_cross_validation': True,\n",
      " 'is_onnx_compatible': False,\n",
      " 'observer': None,\n",
      " 'task': 'classification',\n",
      " 'working_dir': '/mnt/batch/tasks/shared/LS_root/mounts/clusters/notebook177093/code/divorce-prediction-azure-ml'}\n",
      "\n",
      "StandardScalerWrapper\n",
      "{'class_name': 'StandardScaler',\n",
      " 'copy': True,\n",
      " 'module_name': 'sklearn.preprocessing._data',\n",
      " 'with_mean': False,\n",
      " 'with_std': False}\n",
      "\n",
      "RandomForestClassifier\n",
      "{'bootstrap': False,\n",
      " 'ccp_alpha': 0.0,\n",
      " 'class_weight': None,\n",
      " 'criterion': 'entropy',\n",
      " 'max_depth': None,\n",
      " 'max_features': 0.1,\n",
      " 'max_leaf_nodes': None,\n",
      " 'max_samples': None,\n",
      " 'min_impurity_decrease': 0.0,\n",
      " 'min_impurity_split': None,\n",
      " 'min_samples_leaf': 0.01,\n",
      " 'min_samples_split': 0.01,\n",
      " 'min_weight_fraction_leaf': 0.0,\n",
      " 'n_estimators': 50,\n",
      " 'n_jobs': 1,\n",
      " 'oob_score': False,\n",
      " 'random_state': None,\n",
      " 'verbose': 0,\n",
      " 'warm_start': False}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from azureml.train.automl.run import AutoMLRun\n",
    "\n",
    "run_id = ''\n",
    "for step in pipeline_run.get_steps():\n",
    "    run_id = step.id\n",
    "automl_run = AutoMLRun(experiment=experiment, run_id=run_id)\n",
    "best_run, fitted_model = automl_run.get_output()\n",
    "print(best_run)\n",
    "print_model(fitted_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Register best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Register it\n",
    "from azureml.core.run import Run\n",
    "\n",
    "run = Run.get_context()\n",
    "pipeline_run.upload_file(\"./outputs/model.pkl\", best_model_output._path_on_datastore)\n",
    "print(pipeline_run.get_file_names())\n",
    "model = pipeline_run.register_model(model_name=\"auto-ml-model\", model_path=\"./outputs/model.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "authors": [
   {
    "name": "sanpil"
   }
  ],
  "categories": [
   "how-to-use-azureml",
   "machine-learning-pipelines",
   "intro-to-pipelines"
  ],
  "category": "tutorial",
  "compute": [
   "AML Compute"
  ],
  "datasets": [
   "Custom"
  ],
  "deployment": [
   "None"
  ],
  "exclude_from_index": false,
  "framework": [
   "Automated Machine Learning"
  ],
  "friendly_name": "How to use AutoMLStep with AML Pipelines",
  "kernel_info": {
   "name": "python3"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "microsoft": {
   "host": {
    "AzureML": {
     "notebookHasBeenCompleted": true
    }
   }
  },
  "nteract": {
   "version": "nteract-front-end@1.0.0"
  },
  "order_index": 11,
  "star_tag": [
   "featured"
  ],
  "tags": [
   "None"
  ],
  "task": "Demonstrates the use of AutoMLStep"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}