Lumber_Spine_Vgg19.ipynb

{
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.12",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "kaggle": {
      "accelerator": "gpu",
      "dataSources": [
        {
          "sourceId": 71549,
          "databundleVersionId": 8561470,
          "sourceType": "competition"
        },
        {
          "sourceId": 269160,
          "sourceType": "datasetVersion",
          "datasetId": 112702
        }
      ],
      "dockerImageVersionId": 30823,
      "isInternetEnabled": true,
      "language": "python",
      "sourceType": "notebook",
      "isGpuEnabled": true
    },
    "colab": {
      "name": "Lumber Spine - Vgg19",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat_minor": 0,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/masoudshahrian/DeepLearning-Code/blob/main/Lumber_Spine_Vgg19.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "source": [
        "# IMPORTANT: SOME KAGGLE DATA SOURCES ARE PRIVATE\n",
        "# RUN THIS CELL IN ORDER TO IMPORT YOUR KAGGLE DATA SOURCES.\n",
        "import kagglehub\n",
        "kagglehub.login()\n"
      ],
      "metadata": {
        "id": "cNTSQM_7wQIf"
      },
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "source": [
        "# IMPORTANT: RUN THIS CELL IN ORDER TO IMPORT YOUR KAGGLE DATA SOURCES,\n",
        "# THEN FEEL FREE TO DELETE THIS CELL.\n",
        "# NOTE: THIS NOTEBOOK ENVIRONMENT DIFFERS FROM KAGGLE'S PYTHON\n",
        "# ENVIRONMENT SO THERE MAY BE MISSING LIBRARIES USED BY YOUR\n",
        "# NOTEBOOK.\n",
        "\n",
        "rsna_2024_lumbar_spine_degenerative_classification_path = kagglehub.competition_download('rsna-2024-lumbar-spine-degenerative-classification')\n",
        "phuhung273_vgg19dcbb9e9dpth_path = kagglehub.dataset_download('phuhung273/vgg19dcbb9e9dpth')\n",
        "\n",
        "print('Data source import complete.')\n"
      ],
      "metadata": {
        "id": "FTiw2EsrwQIj"
      },
      "cell_type": "code",
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# # This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# # It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
        "# # For example, here's several helpful packages to load\n",
        "\n",
        "# import numpy as np # linear algebra\n",
        "# import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# # Input data files are available in the read-only \"../input/\" directory\n",
        "# # For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
        "\n",
        "# import os\n",
        "# for dirname, _, filenames in os.walk('/kaggle/input'):\n",
        "#     for filename in filenames:\n",
        "#         print(os.path.join(dirname, filename))\n",
        "\n",
        "# # You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\"\n",
        "# # You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
      ],
      "metadata": {
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:04:09.721528Z",
          "iopub.execute_input": "2024-12-25T17:04:09.721945Z",
          "iopub.status.idle": "2024-12-25T17:04:09.726372Z",
          "shell.execute_reply.started": "2024-12-25T17:04:09.721905Z",
          "shell.execute_reply": "2024-12-25T17:04:09.72538Z"
        },
        "id": "vEzlH0P9wQIk"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import os\n",
        "import time\n",
        "import numpy as np\n",
        "import glob\n",
        "import json\n",
        "import collections\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import pydicom as dicom\n",
        "import matplotlib.patches as patches\n",
        "from matplotlib import animation, rc\n",
        "import pandas as pd\n",
        "import pydicom as dicom # dicom\n",
        "import pydicom\n",
        "from pydicom.pixel_data_handlers.util import apply_voi_lut"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:04:09.727884Z",
          "iopub.execute_input": "2024-12-25T17:04:09.728252Z",
          "iopub.status.idle": "2024-12-25T17:04:14.177241Z",
          "shell.execute_reply.started": "2024-12-25T17:04:09.728218Z",
          "shell.execute_reply": "2024-12-25T17:04:14.176606Z"
        },
        "id": "1h3bo5TlwQIl"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# read data\n",
        "train_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/'\n",
        "\n",
        "train  = pd.read_csv(train_path + 'train.csv')\n",
        "label = pd.read_csv(train_path + 'train_label_coordinates.csv')\n",
        "train_desc  = pd.read_csv(train_path + 'train_series_descriptions.csv')\n",
        "test_desc   = pd.read_csv(train_path + 'test_series_descriptions.csv')\n",
        "sub         = pd.read_csv(train_path + 'sample_submission.csv')"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:04:14.178855Z",
          "iopub.execute_input": "2024-12-25T17:04:14.179306Z",
          "iopub.status.idle": "2024-12-25T17:04:14.326329Z",
          "shell.execute_reply.started": "2024-12-25T17:04:14.179284Z",
          "shell.execute_reply": "2024-12-25T17:04:14.325602Z"
        },
        "id": "CqeDPVS5wQIl"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "test_desc.head(5)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:04:14.327493Z",
          "iopub.execute_input": "2024-12-25T17:04:14.327781Z",
          "iopub.status.idle": "2024-12-25T17:04:14.342823Z",
          "shell.execute_reply.started": "2024-12-25T17:04:14.327746Z",
          "shell.execute_reply": "2024-12-25T17:04:14.342008Z"
        },
        "id": "WsxPE6e0wQIm"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "train.head(5)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:04:14.343983Z",
          "iopub.execute_input": "2024-12-25T17:04:14.344292Z",
          "iopub.status.idle": "2024-12-25T17:04:14.366075Z",
          "shell.execute_reply.started": "2024-12-25T17:04:14.344264Z",
          "shell.execute_reply": "2024-12-25T17:04:14.365127Z"
        },
        "id": "iGMDTkv2wQIm"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "train_desc.head(5)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:04:14.367092Z",
          "iopub.execute_input": "2024-12-25T17:04:14.367404Z",
          "iopub.status.idle": "2024-12-25T17:04:14.38234Z",
          "shell.execute_reply.started": "2024-12-25T17:04:14.367376Z",
          "shell.execute_reply": "2024-12-25T17:04:14.381548Z"
        },
        "id": "RulFgFmAwQIn"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Function to generate image paths based on directory structure\n",
        "def generate_image_paths(df, data_dir):\n",
        "    image_paths = []\n",
        "    for study_id, series_id in zip(df['study_id'], df['series_id']):\n",
        "        study_dir = os.path.join(data_dir, str(study_id))\n",
        "        series_dir = os.path.join(study_dir, str(series_id))\n",
        "        images = os.listdir(series_dir)\n",
        "        image_paths.extend([os.path.join(series_dir, img) for img in images])\n",
        "    return image_paths\n",
        "\n",
        "# Generate image paths for train and test data\n",
        "train_image_paths = generate_image_paths(train_desc, f'{train_path}/train_images')\n",
        "test_image_paths = generate_image_paths(test_desc, f'{train_path}/test_images')"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:04:14.383025Z",
          "iopub.execute_input": "2024-12-25T17:04:14.38323Z",
          "iopub.status.idle": "2024-12-25T17:05:22.053801Z",
          "shell.execute_reply.started": "2024-12-25T17:04:14.383213Z",
          "shell.execute_reply": "2024-12-25T17:05:22.053132Z"
        },
        "id": "BzbLl1q1wQIn"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "print(train_image_paths[2])"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:22.056681Z",
          "iopub.execute_input": "2024-12-25T17:05:22.05693Z",
          "iopub.status.idle": "2024-12-25T17:05:22.06135Z",
          "shell.execute_reply.started": "2024-12-25T17:05:22.056911Z",
          "shell.execute_reply": "2024-12-25T17:05:22.060653Z"
        },
        "id": "c17EDh1kwQIo"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "len(train_desc)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:22.063078Z",
          "iopub.execute_input": "2024-12-25T17:05:22.063267Z",
          "iopub.status.idle": "2024-12-25T17:05:22.080035Z",
          "shell.execute_reply.started": "2024-12-25T17:05:22.063251Z",
          "shell.execute_reply": "2024-12-25T17:05:22.079384Z"
        },
        "id": "v6O9ePsOwQIo"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "len(train_image_paths)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:22.080735Z",
          "iopub.execute_input": "2024-12-25T17:05:22.080926Z",
          "iopub.status.idle": "2024-12-25T17:05:22.093171Z",
          "shell.execute_reply.started": "2024-12-25T17:05:22.0809Z",
          "shell.execute_reply": "2024-12-25T17:05:22.092509Z"
        },
        "id": "zdb9EwE8wQIo"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import pydicom\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Function to open and display DICOM images\n",
        "def display_dicom_images(image_paths):\n",
        "    plt.figure(figsize=(15, 5))  # Adjust figure size if needed\n",
        "    for i, path in enumerate(image_paths[:3]):\n",
        "        ds = pydicom.dcmread(path)\n",
        "        plt.subplot(1, 3, i+1)\n",
        "        plt.imshow(ds.pixel_array, cmap=plt.cm.bone)\n",
        "        plt.title(f\"Image {i+1}\")\n",
        "        plt.axis('off')\n",
        "    plt.show()\n",
        "\n",
        "# Display the first three DICOM images\n",
        "display_dicom_images(train_image_paths)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:22.093913Z",
          "iopub.execute_input": "2024-12-25T17:05:22.094145Z",
          "execution_failed": "2024-12-26T08:22:18.132Z"
        },
        "id": "5VCd2fS_wQIp"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import pydicom\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "\n",
        "# Function to open and display DICOM images along with coordinates\n",
        "def display_dicom_with_coordinates(image_paths, label_df):\n",
        "    fig, axs = plt.subplots(1, len(image_paths), figsize=(18, 6))\n",
        "\n",
        "    for idx, path in enumerate(image_paths):  # Display images\n",
        "        study_id = int(path.split('/')[-3])\n",
        "        series_id = int(path.split('/')[-2])\n",
        "\n",
        "        # Filter label coordinates for the current study and series\n",
        "        filtered_labels = label_df[(label_df['study_id'] == study_id) & (label_df['series_id'] == series_id)]\n",
        "\n",
        "        # Read DICOM image\n",
        "        ds = pydicom.dcmread(path)\n",
        "\n",
        "        # Plot DICOM image\n",
        "        axs[idx].imshow(ds.pixel_array, cmap='gray')\n",
        "        axs[idx].set_title(f\"Study ID: {study_id}, Series ID: {series_id}\")\n",
        "        axs[idx].axis('off')\n",
        "\n",
        "        # Plot coordinates\n",
        "        for _, row in filtered_labels.iterrows():\n",
        "            axs[idx].plot(row['x'], row['y'], 'ro', markersize=5)\n",
        "\n",
        "    plt.tight_layout()\n",
        "    plt.show()\n",
        "\n",
        "# Load DICOM files from a folder\n",
        "def load_dicom_files(path_to_folder):\n",
        "    files = [os.path.join(path_to_folder, f) for f in os.listdir(path_to_folder) if f.endswith('.dcm')]\n",
        "    files.sort(key=lambda x: int(os.path.splitext(os.path.basename(x))[0].split('-')[-1]))\n",
        "    return files\n",
        "\n",
        "# Display DICOM images with coordinates\n",
        "study_id = \"100206310\"\n",
        "study_folder = f'{train_path}/train_images/{study_id}'\n",
        "\n",
        "image_paths = []\n",
        "for series_folder in os.listdir(study_folder):\n",
        "    series_folder_path = os.path.join(study_folder, series_folder)\n",
        "    dicom_files = load_dicom_files(series_folder_path)\n",
        "    if dicom_files:\n",
        "        image_paths.append(dicom_files[0])  # Add the first image from each series\n",
        "\n",
        "\n",
        "display_dicom_with_coordinates(image_paths, label)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.idle": "2024-12-25T17:05:23.536867Z",
          "shell.execute_reply.started": "2024-12-25T17:05:22.723348Z",
          "shell.execute_reply": "2024-12-25T17:05:23.535771Z"
        },
        "id": "jWxH5LduwQIp"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Data Preprocessing"
      ],
      "metadata": {
        "id": "F-a5P69qwQIp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define function to reshape a single row of the DataFrame\n",
        "def reshape_row(row):\n",
        "    data = {'study_id': [], 'condition': [], 'level': [], 'severity': []}\n",
        "\n",
        "    for column, value in row.items():\n",
        "        if column not in ['study_id', 'series_id', 'instance_number', 'x', 'y', 'series_description']:\n",
        "            parts = column.split('_')\n",
        "            condition = ' '.join([word.capitalize() for word in parts[:-2]])\n",
        "            level = parts[-2].capitalize() + '/' + parts[-1].capitalize()\n",
        "            data['study_id'].append(row['study_id'])\n",
        "            data['condition'].append(condition)\n",
        "            data['level'].append(level)\n",
        "            data['severity'].append(value)\n",
        "\n",
        "    return pd.DataFrame(data)\n",
        "\n",
        "# Reshape the DataFrame for all rows\n",
        "new_train_df = pd.concat([reshape_row(row) for _, row in train.iterrows()], ignore_index=True)\n",
        "\n",
        "# Display the first few rows of the reshaped dataframe\n",
        "new_train_df.head(5)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:23.537857Z",
          "iopub.execute_input": "2024-12-25T17:05:23.538275Z",
          "iopub.status.idle": "2024-12-25T17:05:24.554839Z",
          "shell.execute_reply.started": "2024-12-25T17:05:23.538239Z",
          "shell.execute_reply": "2024-12-25T17:05:24.554054Z"
        },
        "id": "1pqFfz7nwQIq"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Print columns in a neat way\n",
        "print(\"\\nColumns in new_train_df:\")\n",
        "print(\",\".join(new_train_df.columns))\n",
        "\n",
        "print(\"\\nColumns in label:\")\n",
        "print(\",\".join(label.columns))\n",
        "\n",
        "print(\"\\nColumns in test_desc:\")\n",
        "print(\",\".join(test_desc.columns))\n",
        "\n",
        "print(\"\\nColumns in sub:\")\n",
        "print(\",\".join(sub.columns))"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:24.555651Z",
          "iopub.execute_input": "2024-12-25T17:05:24.555944Z",
          "iopub.status.idle": "2024-12-25T17:05:24.562095Z",
          "shell.execute_reply.started": "2024-12-25T17:05:24.555919Z",
          "shell.execute_reply": "2024-12-25T17:05:24.561423Z"
        },
        "id": "iggb0F5TwQIq"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Merge the dataframes on the common columns\n",
        "merged_df = pd.merge(new_train_df, label, on=['study_id', 'condition', 'level'], how='inner')\n",
        "# Merge the dataframes on the common column 'series_id'\n",
        "final_merged_df = pd.merge(merged_df, train_desc, on='series_id', how='inner')"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:24.562812Z",
          "iopub.execute_input": "2024-12-25T17:05:24.563053Z",
          "iopub.status.idle": "2024-12-25T17:05:24.625511Z",
          "shell.execute_reply.started": "2024-12-25T17:05:24.56303Z",
          "shell.execute_reply": "2024-12-25T17:05:24.624904Z"
        },
        "id": "V5HW4jLrwQIq"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Merge the dataframes on the common column 'series_id'\n",
        "final_merged_df = pd.merge(merged_df, train_desc, on=['series_id','study_id'], how='inner')\n",
        "# Display the first few rows of the final merged dataframe\n",
        "final_merged_df.head(5)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:24.62635Z",
          "iopub.execute_input": "2024-12-25T17:05:24.626662Z",
          "iopub.status.idle": "2024-12-25T17:05:24.650963Z",
          "shell.execute_reply.started": "2024-12-25T17:05:24.626616Z",
          "shell.execute_reply": "2024-12-25T17:05:24.650123Z"
        },
        "id": "dbuxNaDRwQIr"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "final_merged_df[final_merged_df['study_id'] == 100206310].sort_values(['x','y'],ascending = True)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:24.651868Z",
          "iopub.execute_input": "2024-12-25T17:05:24.652188Z",
          "iopub.status.idle": "2024-12-25T17:05:24.668871Z",
          "shell.execute_reply.started": "2024-12-25T17:05:24.652156Z",
          "shell.execute_reply": "2024-12-25T17:05:24.668159Z"
        },
        "id": "L_TaqkZ1wQIr"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "final_merged_df[final_merged_df['series_id'] == 1012284084].sort_values(\"instance_number\")"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:24.669731Z",
          "iopub.execute_input": "2024-12-25T17:05:24.670006Z",
          "iopub.status.idle": "2024-12-25T17:05:24.682121Z",
          "shell.execute_reply.started": "2024-12-25T17:05:24.669986Z",
          "shell.execute_reply": "2024-12-25T17:05:24.681371Z"
        },
        "id": "Uv4ebrvSwQIr"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now, we can see what the data represents\n",
        "\n",
        "Series ID 1012284084 contains 60 images, and how each image maps to each level and condition"
      ],
      "metadata": {
        "id": "dPqTLYUcwQIr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Filter the dataframe for the given study_id and sort by instance_number\n",
        "filtered_df = final_merged_df[final_merged_df['study_id'] == 1013589491].sort_values(\"instance_number\")\n",
        "\n",
        "# Display the resulting dataframe\n",
        "filtered_df"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:24.682936Z",
          "iopub.execute_input": "2024-12-25T17:05:24.683228Z",
          "iopub.status.idle": "2024-12-25T17:05:24.704528Z",
          "shell.execute_reply.started": "2024-12-25T17:05:24.683201Z",
          "shell.execute_reply": "2024-12-25T17:05:24.703822Z"
        },
        "id": "e8U9ddLxwQIr"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Sort final_merged_df by study_id, series_id, and series_description\n",
        "sorted_final_merged_df = final_merged_df[final_merged_df['study_id'] == 1013589491].sort_values(by=['series_id', 'series_description', 'instance_number'])\n",
        "sorted_final_merged_df"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:24.705312Z",
          "iopub.execute_input": "2024-12-25T17:05:24.705542Z",
          "iopub.status.idle": "2024-12-25T17:05:24.732213Z",
          "shell.execute_reply.started": "2024-12-25T17:05:24.705511Z",
          "shell.execute_reply": "2024-12-25T17:05:24.731497Z"
        },
        "id": "OkFDI4m2wQIr"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "source": [
        "We see that, <br>\n",
        "Saggital T1 images map to Neural Foraminal Narrowing <br>\n",
        "Axial T2 images map to Subarticular Stenosis <br>\n",
        "Saggital T2/STIR map to Canal Stenosis <br>"
      ],
      "metadata": {
        "id": "INqDUj9PwQIr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "\n",
        "# Create the row_id column\n",
        "final_merged_df['row_id'] = (\n",
        "    final_merged_df['study_id'].astype(str) + '_' +\n",
        "    final_merged_df['condition'].str.lower().str.replace(' ', '_') + '_' +\n",
        "    final_merged_df['level'].str.lower().str.replace('/', '_')\n",
        ")\n",
        "\n",
        "# Create the image_path column\n",
        "final_merged_df['image_path'] = (\n",
        "    f'{train_path}/train_images/' +\n",
        "    final_merged_df['study_id'].astype(str) + '/' +\n",
        "    final_merged_df['series_id'].astype(str) + '/' +\n",
        "    final_merged_df['instance_number'].astype(str) + '.dcm'\n",
        ")\n",
        "\n",
        "# Note: Check image path, since there's 1 instance id, for 1 image, but there's many more images other than the ones labelled in the instance ID.\n",
        "\n",
        "# Display the updated dataframe\n",
        "final_merged_df.head(5)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:24.733154Z",
          "iopub.execute_input": "2024-12-25T17:05:24.733432Z",
          "iopub.status.idle": "2024-12-25T17:05:24.917843Z",
          "shell.execute_reply.started": "2024-12-25T17:05:24.733404Z",
          "shell.execute_reply": "2024-12-25T17:05:24.916992Z"
        },
        "id": "muggdgcQwQIr"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "final_merged_df[final_merged_df[\"severity\"] == \"Normal/Mild\"].value_counts().sum()"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:24.91874Z",
          "iopub.execute_input": "2024-12-25T17:05:24.919058Z",
          "iopub.status.idle": "2024-12-25T17:05:25.017557Z",
          "shell.execute_reply.started": "2024-12-25T17:05:24.919029Z",
          "shell.execute_reply": "2024-12-25T17:05:25.016825Z"
        },
        "id": "0DbBt1BcwQIs"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "final_merged_df[final_merged_df[\"severity\"] == \"Moderate\"].value_counts().sum()"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:25.02197Z",
          "iopub.execute_input": "2024-12-25T17:05:25.022184Z",
          "iopub.status.idle": "2024-12-25T17:05:25.056049Z",
          "shell.execute_reply.started": "2024-12-25T17:05:25.022166Z",
          "shell.execute_reply": "2024-12-25T17:05:25.05537Z"
        },
        "id": "jOlJHO7qwQIs"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "final_merged_df[final_merged_df[\"severity\"] == \"Severe\"].value_counts().sum()"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:25.058033Z",
          "iopub.execute_input": "2024-12-25T17:05:25.058259Z",
          "iopub.status.idle": "2024-12-25T17:05:25.078529Z",
          "shell.execute_reply.started": "2024-12-25T17:05:25.058232Z",
          "shell.execute_reply": "2024-12-25T17:05:25.077904Z"
        },
        "id": "-5HrThk8wQIs"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the base path for test images\n",
        "base_path = '/kaggle/input/rsna-2024-lumbar-spine-degenerative-classification/test_images/'\n",
        "\n",
        "# Function to get image paths for a series\n",
        "def get_image_paths(row):\n",
        "    series_path = os.path.join(base_path, str(row['study_id']), str(row['series_id']))\n",
        "    if os.path.exists(series_path):\n",
        "        return [os.path.join(series_path, f) for f in os.listdir(series_path) if os.path.isfile(os.path.join(series_path, f))]\n",
        "    return []\n",
        "\n",
        "# Mapping of series_description to conditions\n",
        "condition_mapping = {\n",
        "    'Sagittal T1': {'left': 'left_neural_foraminal_narrowing', 'right': 'right_neural_foraminal_narrowing'},\n",
        "    'Axial T2': {'left': 'left_subarticular_stenosis', 'right': 'right_subarticular_stenosis'},\n",
        "    'Sagittal T2/STIR': 'spinal_canal_stenosis'\n",
        "}\n",
        "\n",
        "# Create a list to store the expanded rows\n",
        "expanded_rows = []\n",
        "\n",
        "# Expand the dataframe by adding new rows for each file path\n",
        "for index, row in test_desc.iterrows():\n",
        "    image_paths = get_image_paths(row)\n",
        "    conditions = condition_mapping.get(row['series_description'], {})\n",
        "    if isinstance(conditions, str):  # Single condition\n",
        "        conditions = {'left': conditions, 'right': conditions}\n",
        "    for side, condition in conditions.items():\n",
        "        for image_path in image_paths:\n",
        "            expanded_rows.append({\n",
        "                'study_id': row['study_id'],\n",
        "                'series_id': row['series_id'],\n",
        "                'series_description': row['series_description'],\n",
        "                'image_path': image_path,\n",
        "                'condition': condition,\n",
        "                'row_id': f\"{row['study_id']}_{condition}\"\n",
        "            })\n",
        "\n",
        "# Create a new dataframe from the expanded rows\n",
        "expanded_test_desc = pd.DataFrame(expanded_rows)\n",
        "\n",
        "# Display the resulting dataframe\n",
        "expanded_test_desc.head(5)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:25.079225Z",
          "iopub.execute_input": "2024-12-25T17:05:25.079497Z",
          "iopub.status.idle": "2024-12-25T17:05:25.194554Z",
          "shell.execute_reply.started": "2024-12-25T17:05:25.079476Z",
          "shell.execute_reply": "2024-12-25T17:05:25.193828Z"
        },
        "id": "7RqATBHmwQIt"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# change severity column labels\n",
        "#Normal/Mild': 'normal_mild', 'Moderate': 'moderate', 'Severe': 'severe'}\n",
        "final_merged_df['severity'] = final_merged_df['severity'].map({'Normal/Mild': 'normal_mild', 'Moderate': 'moderate', 'Severe': 'severe'})"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:25.195341Z",
          "iopub.execute_input": "2024-12-25T17:05:25.195597Z",
          "iopub.status.idle": "2024-12-25T17:05:25.203034Z",
          "shell.execute_reply.started": "2024-12-25T17:05:25.195564Z",
          "shell.execute_reply": "2024-12-25T17:05:25.202146Z"
        },
        "id": "XckFOT1owQIu"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "test_data = expanded_test_desc\n",
        "train_data = final_merged_df"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:25.204111Z",
          "iopub.execute_input": "2024-12-25T17:05:25.204344Z",
          "iopub.status.idle": "2024-12-25T17:05:25.215817Z",
          "shell.execute_reply.started": "2024-12-25T17:05:25.204325Z",
          "shell.execute_reply": "2024-12-25T17:05:25.215019Z"
        },
        "id": "uyDtrWTlwQIu"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "train_data.head(10)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:25.216577Z",
          "iopub.execute_input": "2024-12-25T17:05:25.216859Z",
          "iopub.status.idle": "2024-12-25T17:05:25.237809Z",
          "shell.execute_reply.started": "2024-12-25T17:05:25.216827Z",
          "shell.execute_reply": "2024-12-25T17:05:25.237014Z"
        },
        "id": "HUCmrbOhwQIv"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "test_data.head(10)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:25.238661Z",
          "iopub.execute_input": "2024-12-25T17:05:25.238979Z",
          "iopub.status.idle": "2024-12-25T17:05:25.255514Z",
          "shell.execute_reply.started": "2024-12-25T17:05:25.238946Z",
          "shell.execute_reply": "2024-12-25T17:05:25.254788Z"
        },
        "id": "XJzgKNXqwQIv"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "train_data.shape"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:25.256196Z",
          "iopub.execute_input": "2024-12-25T17:05:25.256453Z",
          "iopub.status.idle": "2024-12-25T17:05:25.268353Z",
          "shell.execute_reply.started": "2024-12-25T17:05:25.256429Z",
          "shell.execute_reply": "2024-12-25T17:05:25.267616Z"
        },
        "id": "QaAMTTVOwQIv"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "# Define a function to check if a path exists\n",
        "def check_exists(path):\n",
        "    return os.path.exists(path)\n",
        "\n",
        "# Define a function to check if a study ID directory exists\n",
        "def check_study_id(row):\n",
        "    study_id = row['study_id']\n",
        "    path = f'{train_path}/train_images/{study_id}'\n",
        "    return check_exists(path)\n",
        "\n",
        "# Define a function to check if a series ID directory exists\n",
        "def check_series_id(row):\n",
        "    study_id = row['study_id']\n",
        "    series_id = row['series_id']\n",
        "    path = f'{train_path}/train_images/{study_id}/{series_id}'\n",
        "    return check_exists(path)\n",
        "\n",
        "# Define a function to check if an image file exists\n",
        "def check_image_exists(row):\n",
        "    image_path = row['image_path']\n",
        "    return check_exists(image_path)\n",
        "\n",
        "# Apply the functions to the train_data dataframe\n",
        "train_data['study_id_exists'] = train_data.apply(check_study_id, axis=1)\n",
        "train_data['series_id_exists'] = train_data.apply(check_series_id, axis=1)\n",
        "train_data['image_exists'] = train_data.apply(check_image_exists, axis=1)\n",
        "\n",
        "# Filter train_data\n",
        "train_data = train_data[(train_data['study_id_exists']) & (train_data['series_id_exists']) & (train_data['image_exists'])]"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:25.269146Z",
          "iopub.execute_input": "2024-12-25T17:05:25.269416Z",
          "iopub.status.idle": "2024-12-25T17:05:56.241689Z",
          "shell.execute_reply.started": "2024-12-25T17:05:25.269391Z",
          "shell.execute_reply": "2024-12-25T17:05:56.240995Z"
        },
        "id": "pxLn2clOwQIv"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "train_data.shape"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:56.242426Z",
          "iopub.execute_input": "2024-12-25T17:05:56.242727Z",
          "iopub.status.idle": "2024-12-25T17:05:56.248148Z",
          "shell.execute_reply.started": "2024-12-25T17:05:56.242695Z",
          "shell.execute_reply": "2024-12-25T17:05:56.247499Z"
        },
        "id": "NBjBIJTuwQIv"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "train_data.head(3)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:56.248889Z",
          "iopub.execute_input": "2024-12-25T17:05:56.249087Z",
          "iopub.status.idle": "2024-12-25T17:05:56.273511Z",
          "shell.execute_reply.started": "2024-12-25T17:05:56.249071Z",
          "shell.execute_reply": "2024-12-25T17:05:56.272753Z"
        },
        "id": "EWeDicftwQIv"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "def load_dicom(path):\n",
        "    dicom = pydicom.read_file(path)\n",
        "    data = dicom.pixel_array\n",
        "    data = data - np.min(data)\n",
        "    if np.max(data) != 0:\n",
        "        data = data / np.max(data)\n",
        "    data = (data * 255).astype(np.uint8)\n",
        "    return data"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:56.274363Z",
          "iopub.execute_input": "2024-12-25T17:05:56.274605Z",
          "iopub.status.idle": "2024-12-25T17:05:56.289519Z",
          "shell.execute_reply.started": "2024-12-25T17:05:56.274581Z",
          "shell.execute_reply": "2024-12-25T17:05:56.288668Z"
        },
        "id": "sJRCQMCOwQIw"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import pydicom\n",
        "import numpy as np\n",
        "import random\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "def load_dicom(path):\n",
        "    dicom = pydicom.dcmread(path)  # استفاده از dcmread به جای read_file\n",
        "    data = dicom.pixel_array\n",
        "    data = data - np.min(data)\n",
        "    if np.max(data) != 0:\n",
        "        data = data / np.max(data)\n",
        "    data = (data * 255).astype(np.uint8)\n",
        "    return data\n",
        "\n",
        "# فرض بر این است که train_data تعریف شده و شامل 'image_path' و 'row_id' است\n",
        "images = []\n",
        "row_ids = []\n",
        "selected_indices = random.sample(range(len(train_data)), 2)\n",
        "for i in selected_indices:\n",
        "    image = load_dicom(train_data['image_path'][i])\n",
        "    images.append(image)\n",
        "    row_ids.append(train_data['row_id'][i])\n",
        "\n",
        "# رسم تصاویر\n",
        "fig, ax = plt.subplots(1, 2, figsize=(8, 4))\n",
        "for i in range(2):\n",
        "    ax[i].imshow(images[i], cmap='gray')\n",
        "    ax[i].set_title(f'Row ID: {row_ids[i]}', fontsize=8)\n",
        "    ax[i].axis('off')\n",
        "plt.tight_layout()\n",
        "plt.show()"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:16:46.441814Z",
          "iopub.execute_input": "2024-12-25T17:16:46.442171Z",
          "iopub.status.idle": "2024-12-25T17:16:46.853878Z",
          "shell.execute_reply.started": "2024-12-25T17:16:46.442148Z",
          "shell.execute_reply": "2024-12-25T17:16:46.852936Z"
        },
        "id": "ByXiPumJwQI1"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Load images randomly\n",
        "import random\n",
        "images = []\n",
        "row_ids = []\n",
        "selected_indices = random.sample(range(len(train_data)), 2)\n",
        "for i in selected_indices:\n",
        "    image = load_dicom(train_data['image_path'][i])\n",
        "    images.append(image)\n",
        "    row_ids.append(train_data['row_id'][i])\n",
        "\n",
        "# Plot images\n",
        "fig, ax = plt.subplots(1, 2, figsize=(8, 4))\n",
        "for i in range(2):\n",
        "    ax[i].imshow(images[i], cmap='gray')\n",
        "    ax[i].set_title(f'Row ID: {row_ids[i]}', fontsize=8)\n",
        "    ax[i].axis('off')\n",
        "plt.tight_layout()\n",
        "plt.show()"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:16:51.668077Z",
          "iopub.execute_input": "2024-12-25T17:16:51.668363Z",
          "iopub.status.idle": "2024-12-25T17:16:52.043891Z",
          "shell.execute_reply.started": "2024-12-25T17:16:51.668343Z",
          "shell.execute_reply": "2024-12-25T17:16:52.042923Z"
        },
        "id": "SwmpgmEZwQI2"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Loading data"
      ],
      "metadata": {
        "id": "fNOp_W7ZwQI2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#for one hot encoding\n",
        "#train_data[['normal_mild', 'severe', 'moderate']] = train_data[['normal_mild', 'severe', 'moderate']].astype(int)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:56.335694Z",
          "iopub.status.idle": "2024-12-25T17:05:56.335998Z",
          "shell.execute_reply": "2024-12-25T17:05:56.335886Z"
        },
        "id": "x0iRgbdNwQI2"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "train_data"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:16:59.8076Z",
          "iopub.execute_input": "2024-12-25T17:16:59.807994Z",
          "iopub.status.idle": "2024-12-25T17:16:59.825766Z",
          "shell.execute_reply.started": "2024-12-25T17:16:59.807967Z",
          "shell.execute_reply": "2024-12-25T17:16:59.824938Z"
        },
        "id": "7j7awV5LwQI2"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "train_data = train_data.dropna()"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:17:02.503635Z",
          "iopub.execute_input": "2024-12-25T17:17:02.503974Z",
          "iopub.status.idle": "2024-12-25T17:17:02.528807Z",
          "shell.execute_reply.started": "2024-12-25T17:17:02.503952Z",
          "shell.execute_reply": "2024-12-25T17:17:02.528127Z"
        },
        "id": "gJSq76hgwQI3"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "import torchvision.transforms as transforms\n",
        "import torch\n",
        "import torch.optim.lr_scheduler as lr_scheduler\n",
        "from tqdm import tqdm\n",
        "\n",
        "# Define a custom dataset class\n",
        "class CustomDataset(Dataset):\n",
        "    def __init__(self, dataframe, transform=None):\n",
        "        self.dataframe = dataframe\n",
        "        self.transform = transform\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.dataframe)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        image_path = self.dataframe['image_path'][index]\n",
        "        image = load_dicom(image_path)  # Define this function to load your DICOM images\n",
        "        label = self.dataframe['severity'][index]\n",
        "\n",
        "        if self.transform:\n",
        "            image = self.transform(image)\n",
        "\n",
        "        return image, label\n",
        "\n",
        "# Function to create datasets and dataloaders for each series description\n",
        "def create_datasets_and_loaders(df, series_description, transform, batch_size=8):\n",
        "    filtered_df = df[df['series_description'] == series_description]\n",
        "\n",
        "    train_df, val_df = train_test_split(filtered_df, test_size=0.2, random_state=42)\n",
        "    train_df = train_df.reset_index(drop=True)\n",
        "    val_df = val_df.reset_index(drop=True)\n",
        "\n",
        "    train_dataset = CustomDataset(train_df, transform)\n",
        "    val_dataset = CustomDataset(val_df, transform)\n",
        "\n",
        "    trainloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
        "    valloader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)\n",
        "\n",
        "    return trainloader, valloader, len(train_df), len(val_df)\n",
        "\n",
        "# Define the transforms\n",
        "transform = transforms.Compose([\n",
        "    transforms.Lambda(lambda x: (x * 255).astype(np.uint8)),  # Convert back to uint8 for PIL\n",
        "    transforms.ToPILImage(),\n",
        "    transforms.Resize((224, 224)),\n",
        "    transforms.Grayscale(num_output_channels=3),\n",
        "    transforms.ToTensor(),\n",
        "])\n",
        "\n",
        "# Create dataloaders for each series description\n",
        "dataloaders = {}\n",
        "lengths = {}\n",
        "\n",
        "trainloader_t1, valloader_t1, len_train_t1, len_val_t1 = create_datasets_and_loaders(train_data, 'Sagittal T1', transform)\n",
        "trainloader_t2, valloader_t2, len_train_t2, len_val_t2 = create_datasets_and_loaders(train_data, 'Axial T2', transform)\n",
        "trainloader_t2stir, valloader_t2stir, len_train_t2stir, len_val_t2stir = create_datasets_and_loaders(train_data, 'Sagittal T2/STIR', transform)\n",
        "\n",
        "dataloaders['Sagittal T1'] = (trainloader_t1, valloader_t1)\n",
        "dataloaders['Axial T2'] = (trainloader_t2, valloader_t2)\n",
        "dataloaders['Sagittal T2/STIR'] = (trainloader_t2stir, valloader_t2stir)\n",
        "\n",
        "lengths['Sagittal T1'] = (len_train_t1, len_val_t1)\n",
        "lengths['Axial T2'] = (len_train_t2, len_val_t2)\n",
        "lengths['Sagittal T2/STIR'] = (len_train_t2stir, len_val_t2stir)\n",
        "\n",
        "# Dictionary mapping labels to indices\n",
        "label_map = {'Mild': 0, 'Moderate': 1, 'Severe': 2}"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:17:04.334321Z",
          "iopub.execute_input": "2024-12-25T17:17:04.334607Z",
          "iopub.status.idle": "2024-12-25T17:17:05.648497Z",
          "shell.execute_reply.started": "2024-12-25T17:17:04.334587Z",
          "shell.execute_reply": "2024-12-25T17:17:05.647835Z"
        },
        "id": "0i6zFmqUwQI3"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Function to visualize a batch of images\n",
        "def visualize_batch(dataloader):\n",
        "    images, labels = next(iter(dataloader))\n",
        "    fig, axes = plt.subplots(1, len(images), figsize=(20, 5))\n",
        "    for i, (img, lbl) in enumerate(zip(images, labels)):\n",
        "        ax = axes[i]\n",
        "        img = img.permute(1, 2, 0)  # Convert to HWC for visualization\n",
        "        ax.imshow(img)\n",
        "        ax.set_title(f\"Label: {lbl}\")\n",
        "        ax.axis('off')\n",
        "    plt.show()\n",
        "\n",
        "# Visualize samples from each dataloader\n",
        "print(\"Visualizing Sagittal T1 samples\")\n",
        "visualize_batch(trainloader_t1)\n",
        "print(\"Visualizing Axial T2 samples\")\n",
        "visualize_batch(trainloader_t2)\n",
        "print(\"Visualizing Sagittal T2/STIR samples\")\n",
        "visualize_batch(trainloader_t2stir)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:17:13.816661Z",
          "iopub.execute_input": "2024-12-25T17:17:13.817194Z",
          "iopub.status.idle": "2024-12-25T17:17:15.962308Z",
          "shell.execute_reply.started": "2024-12-25T17:17:13.817164Z",
          "shell.execute_reply": "2024-12-25T17:17:15.961271Z"
        },
        "id": "zmMwA8DqwQI3"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "image, label = next(iter(trainloader_t2))\n",
        "sample = image[1].permute(1, 2, 0)  #sample\n",
        "\n",
        "# Plot images\n",
        "plt.figsize=(8, 4)\n",
        "plt.imshow(images[0], cmap='gray')\n",
        "plt.title(label[0])\n",
        "plt.axis('off')\n",
        "plt.tight_layout()\n",
        "plt.show()"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:17:20.322987Z",
          "iopub.execute_input": "2024-12-25T17:17:20.323278Z",
          "iopub.status.idle": "2024-12-25T17:17:20.637291Z",
          "shell.execute_reply.started": "2024-12-25T17:17:20.323257Z",
          "shell.execute_reply": "2024-12-25T17:17:20.636493Z"
        },
        "id": "2OItDruiwQI3"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model"
      ],
      "metadata": {
        "id": "mpIXrolhwQI3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# import torch\n",
        "# import torch.nn as nn\n",
        "# import torchvision.models as models\n",
        "# from torch.utils.data import DataLoader, Dataset\n",
        "# from sklearn.model_selection import train_test_split\n",
        "# import pandas as pd\n",
        "# from tqdm import tqdm\n",
        "# from copy import deepcopy\n",
        "# import torchvision.models as models\n",
        "\n",
        "# # تعیین دستگاه (CPU یا GPU)\n",
        "# device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "# class CustomVGG16(nn.Module):\n",
        "#     def __init__(self, num_classes=3, pretrained_weights=None):\n",
        "#         super(CustomVGG16, self).__init__()\n",
        "#         # بارگذاری مدل VGG16 بدون وزن‌ها\n",
        "#         self.model = models.vgg16(weights=None)\n",
        "\n",
        "#         # تغییر لایه نهایی به تعداد کلاس‌های مورد نظر\n",
        "#         num_ftrs = self.model.classifier[6].in_features\n",
        "#         self.model.classifier[6] = nn.Linear(num_ftrs, num_classes)\n",
        "\n",
        "#         # بارگذاری وزن‌های محلی در صورت وجود\n",
        "#         if pretrained_weights:\n",
        "#             # Load the state dict\n",
        "#             state_dict = torch.load(pretrained_weights, map_location=device)\n",
        "\n",
        "#             # Remove the classifier layer from the state_dict\n",
        "#             if 'classifier.6.weight' in state_dict:\n",
        "#                 del state_dict['classifier.6.weight']\n",
        "#             if 'classifier.6.bias' in state_dict:\n",
        "#                 del state_dict['classifier.6.bias']\n",
        "\n",
        "#             # Load the remaining state dict\n",
        "#             self.model.load_state_dict(state_dict, strict=False)\n",
        "\n",
        "#     def forward(self, x):\n",
        "#         return self.model(x)\n",
        "\n",
        "# # مسیر به فایل وزن‌های محلی\n",
        "# weights_path = '/kaggle/input/vgg16/pytorch/v1/1/VGG16_model.pth'\n",
        "\n",
        "# # مدل‌ها را ایجاد کنید\n",
        "# sagittal_t1_model = CustomVGG16(num_classes=3, pretrained_weights=weights_path).to(device)\n",
        "# axial_t2_model = CustomVGG16(num_classes=3, pretrained_weights=weights_path).to(device)\n",
        "# sagittal_t2stir_model = CustomVGG16(num_classes=3, pretrained_weights=weights_path).to(device)\n",
        "\n",
        "# # به صورت اختیاری لایه‌های اولیه را قفل کنید\n",
        "# for model in [sagittal_t1_model, axial_t2_model, sagittal_t2stir_model]:\n",
        "#     for param in model.model.features.parameters():\n",
        "#         param.requires_grad = False\n",
        "\n",
        "# # باز کردن قفل لایه‌های کاملاً متصل نهایی\n",
        "# for model in [sagittal_t1_model, axial_t2_model, sagittal_t2stir_model]:\n",
        "#     for param in model.model.classifier.parameters():\n",
        "#         param.requires_grad = True\n",
        "\n",
        "# # پارامترهای آموزشی\n",
        "# criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "# # ایجاد بهینه‌سازهای جداگانه برای هر مدل\n",
        "# optimizer_sagittal_t1 = torch.optim.Adam(sagittal_t1_model.model.classifier.parameters(), lr=0.001)\n",
        "# optimizer_axial_t2 = torch.optim.Adam(axial_t2_model.model.classifier.parameters(), lr=0.001)\n",
        "# optimizer_sagittal_t2stir = torch.optim.Adam(sagittal_t2stir_model.model.classifier.parameters(), lr=0.001)\n",
        "\n",
        "# # ذخیره مدل‌ها و بهینه‌سازها در دیکشنری‌ها برای دسترسی آسان\n",
        "# models = {\n",
        "#     'Sagittal T1': sagittal_t1_model,\n",
        "#     'Axial T2': axial_t2_model,\n",
        "#     'Sagittal T2/STIR': sagittal_t2stir_model,\n",
        "# }\n",
        "# optimizers = {\n",
        "#     'Sagittal T1': optimizer_sagittal_t1,\n",
        "#     'Axial T2': optimizer_axial_t2,\n",
        "#     'Sagittal T2/STIR': optimizer_sagittal_t2stir,\n",
        "# }"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:05:56.343549Z",
          "iopub.status.idle": "2024-12-25T17:05:56.343934Z",
          "shell.execute_reply": "2024-12-25T17:05:56.343773Z"
        },
        "id": "du2KPM8UwQI4"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torchvision.models as models\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from sklearn.model_selection import train_test_split\n",
        "import pandas as pd\n",
        "from tqdm import tqdm\n",
        "from copy import deepcopy\n",
        "\n",
        "# تعیین دستگاه (CPU یا GPU)\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "class CustomVGG19(nn.Module):\n",
        "    def __init__(self, num_classes=3, pretrained_weights=None):\n",
        "        super(CustomVGG19, self).__init__()\n",
        "        # بارگذاری مدل VGG19 بدون وزن‌ها\n",
        "        self.model = models.vgg19(weights=None)\n",
        "\n",
        "        # بارگذاری وزن‌های محلی در صورت وجود\n",
        "        if pretrained_weights:\n",
        "            self.model.load_state_dict(torch.load(pretrained_weights, map_location=device))\n",
        "\n",
        "        # تغییر لایه نهایی به تعداد کلاس‌های مورد نظر\n",
        "        num_ftrs = self.model.classifier[6].in_features\n",
        "        self.model.classifier[6] = nn.Linear(num_ftrs, num_classes)\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.model(x)\n",
        "\n",
        "    def unfreeze_model(self):\n",
        "        # باز کردن قفل لایه‌های نهایی، در حالی که لایه‌های BatchNorm را قفل نگه می‌داریم\n",
        "        for layer in list(self.model.features.children())[-20:]:\n",
        "            if not isinstance(layer, nn.BatchNorm2d):\n",
        "                for param in layer.parameters():\n",
        "                    param.requires_grad = True\n",
        "\n",
        "        # باز کردن قفل لایه‌های classifier\n",
        "        for param in self.model.classifier.parameters():\n",
        "            param.requires_grad = True\n",
        "\n",
        "# مسیر به فایل وزن‌های محلی\n",
        "# weights_path = '/kaggle/input/vgg19/pytorch/default/1/vgg19-weights.pth'  # مسیر صحیح را وارد کنید\n",
        "# weights_path ='/kaggle/input/vgg19-pretrained-imagenet1k_v1-pytorch/pytorch/default/1/vgg19-dcbb9e9d.pth'\n",
        "# /kaggle/input/vgg19_pretrained/pytorch/pretrained/1/vgg19-dcbb9e9d (1).pth\n",
        "weights_path ='/kaggle/input/vgg19dcbb9e9dpth/vgg19-dcbb9e9d.pth'\n",
        "# مدل‌ها را ایجاد کنید\n",
        "sagittal_t1_model = CustomVGG19(num_classes=3, pretrained_weights=weights_path).to(device)\n",
        "axial_t2_model = CustomVGG19(num_classes=3, pretrained_weights=weights_path).to(device)\n",
        "sagittal_t2stir_model = CustomVGG19(num_classes=3, pretrained_weights=weights_path).to(device)\n",
        "\n",
        "# به صورت اختیاری لایه‌های اولیه را قفل کنید\n",
        "for model in [sagittal_t1_model, axial_t2_model, sagittal_t2stir_model]:\n",
        "    for param in model.model.features.parameters():\n",
        "        param.requires_grad = False\n",
        "\n",
        "# باز کردن قفل لایه‌های کاملاً متصل نهایی\n",
        "for model in [sagittal_t1_model, axial_t2_model, sagittal_t2stir_model]:\n",
        "    for param in model.model.classifier.parameters():\n",
        "        param.requires_grad = True\n",
        "\n",
        "# پارامترهای آموزشی\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "# ایجاد بهینه‌سازهای جداگانه برای هر مدل\n",
        "optimizer_sagittal_t1 = torch.optim.Adam(sagittal_t1_model.model.classifier.parameters(), lr=0.001)\n",
        "optimizer_axial_t2 = torch.optim.Adam(axial_t2_model.model.classifier.parameters(), lr=0.001)\n",
        "optimizer_sagittal_t2stir = torch.optim.Adam(sagittal_t2stir_model.model.classifier.parameters(), lr=0.001)\n",
        "\n",
        "# ذخیره مدل‌ها و بهینه‌سازها در دیکشنری‌ها برای دسترسی آسان\n",
        "models = {\n",
        "    'Sagittal T1': sagittal_t1_model,\n",
        "    'Axial T2': axial_t2_model,\n",
        "    'Sagittal T2/STIR': sagittal_t2stir_model,\n",
        "}\n",
        "optimizers = {\n",
        "    'Sagittal T1': optimizer_sagittal_t1,\n",
        "    'Axial T2': optimizer_axial_t2,\n",
        "    'Sagittal T2/STIR': optimizer_sagittal_t2stir,\n",
        "}"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:17:27.399499Z",
          "iopub.execute_input": "2024-12-25T17:17:27.399827Z",
          "iopub.status.idle": "2024-12-25T17:17:40.256262Z",
          "shell.execute_reply.started": "2024-12-25T17:17:27.399802Z",
          "shell.execute_reply": "2024-12-25T17:17:40.25543Z"
        },
        "id": "m1knbg7twQI4"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Count trainable parameters\n",
        "trainable_params = sum(p.numel() for p in sagittal_t1_model.parameters() if p.requires_grad)\n",
        "print(f\"Number of parameters: {trainable_params}\")"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:17:47.714599Z",
          "iopub.execute_input": "2024-12-25T17:17:47.714956Z",
          "iopub.status.idle": "2024-12-25T17:17:47.719865Z",
          "shell.execute_reply.started": "2024-12-25T17:17:47.71493Z",
          "shell.execute_reply": "2024-12-25T17:17:47.719057Z"
        },
        "id": "6nffstpewQI4"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Training"
      ],
      "metadata": {
        "id": "4vjWpk42wQI5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "label_map = {'normal_mild': 0, 'moderate': 1, 'severe': 2}"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:17:51.787144Z",
          "iopub.execute_input": "2024-12-25T17:17:51.787441Z",
          "iopub.status.idle": "2024-12-25T17:17:51.79119Z",
          "shell.execute_reply.started": "2024-12-25T17:17:51.787421Z",
          "shell.execute_reply": "2024-12-25T17:17:51.790326Z"
        },
        "id": "gfgRiW_XwQI5"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "for images, labels in trainloader_t2:\n",
        "    labels = torch.tensor([label_map[label] for label in labels])\n",
        "    labels = labels.to(device)\n",
        "    print(labels)\n",
        "    break"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:17:53.166626Z",
          "iopub.execute_input": "2024-12-25T17:17:53.166962Z",
          "iopub.status.idle": "2024-12-25T17:17:53.351199Z",
          "shell.execute_reply.started": "2024-12-25T17:17:53.16694Z",
          "shell.execute_reply": "2024-12-25T17:17:53.35041Z"
        },
        "id": "a7vZxRoEwQI5"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import torch.optim.lr_scheduler as lr_scheduler\n",
        "from copy import deepcopy\n",
        "\n",
        "def train_model(model, trainloader, valloader, len_train, len_val, optimizer, num_epochs=10, patience=3):\n",
        "    # Learning rate scheduler\n",
        "    scheduler = lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)\n",
        "\n",
        "    best_val_acc = 0.0\n",
        "    best_model_wts = deepcopy(model.state_dict())\n",
        "    counter = 0\n",
        "\n",
        "    for epoch in range(num_epochs):\n",
        "        model.train()\n",
        "        train_loss = 0\n",
        "        correct_train = 0\n",
        "\n",
        "        with tqdm(trainloader, unit=\"batch\") as tepoch:\n",
        "            for images, labels in tepoch:\n",
        "                images, labels = images.to(device), torch.tensor([label_map[label] for label in labels]).to(device)\n",
        "                optimizer.zero_grad()\n",
        "                outputs = model(images)\n",
        "                loss = criterion(outputs, labels)\n",
        "                loss.backward()\n",
        "                optimizer.step()\n",
        "                train_loss += loss.item()\n",
        "\n",
        "                probabilities = torch.softmax(outputs, dim=1)\n",
        "                _, predicted = torch.max(probabilities, 1)\n",
        "                correct_train += (predicted == labels).sum().item()\n",
        "\n",
        "                tepoch.set_postfix(epoch=epoch+1)\n",
        "\n",
        "        scheduler.step()\n",
        "\n",
        "        train_loss /= len(trainloader)\n",
        "        train_acc = 100 * correct_train / len_train\n",
        "\n",
        "        model.eval()\n",
        "        val_loss, correct_val = 0, 0\n",
        "        with torch.no_grad():\n",
        "            with tqdm(valloader, unit=\"batch\") as vepoch:\n",
        "                for images, labels in vepoch:\n",
        "                    images, labels = images.to(device), torch.tensor([label_map[label] for label in labels]).to(device)\n",
        "                    outputs = model(images)\n",
        "                    loss = criterion(outputs, labels)\n",
        "                    val_loss += loss.item()\n",
        "\n",
        "                    probabilities = torch.softmax(outputs, dim=1).squeeze(0)\n",
        "                    _, predicted = torch.max(probabilities, 1)\n",
        "                    correct_val += (predicted == labels).sum().item()\n",
        "\n",
        "                    vepoch.set_postfix(epoch=epoch+1)\n",
        "\n",
        "        val_loss /= len(valloader)\n",
        "        val_acc = 100 * correct_val / len_val\n",
        "\n",
        "        print(f\"Epoch {epoch+1}, Train Loss: {train_loss:.4f}, Train Acc: {train_acc:.2f}%, Val Loss: {val_loss:.4f}, Val Acc: {val_acc:.2f}%\")\n",
        "\n",
        "        # Save the best model and check for early stopping\n",
        "        if val_acc > best_val_acc:\n",
        "            best_val_acc = val_acc\n",
        "            best_model_wts = deepcopy(model.state_dict())\n",
        "            counter = 0\n",
        "            torch.save(best_model_wts, f'best_model_{epoch+1}.pth')\n",
        "        else:\n",
        "            counter += 1\n",
        "\n",
        "        # Early stopping\n",
        "        if counter >= patience:\n",
        "            print(f\"Early stopping triggered after {epoch+1} epochs\")\n",
        "            break\n",
        "\n",
        "    # Load best model weights\n",
        "    model.load_state_dict(best_model_wts)\n",
        "    return model, best_val_acc"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:17:56.123302Z",
          "iopub.execute_input": "2024-12-25T17:17:56.123747Z",
          "iopub.status.idle": "2024-12-25T17:17:56.135223Z",
          "shell.execute_reply.started": "2024-12-25T17:17:56.123704Z",
          "shell.execute_reply": "2024-12-25T17:17:56.134129Z"
        },
        "id": "sc_-L_-twQI5"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Training all models\n",
        "for desc, model in models.items():\n",
        "    if desc == 'Sagittal T1':\n",
        "        trainloader, valloader, len_train, len_val = trainloader_t1, valloader_t1, len_train_t1, len_val_t1\n",
        "    elif desc == 'Axial T2':\n",
        "        trainloader, valloader, len_train, len_val = trainloader_t2, valloader_t2, len_train_t2, len_val_t2\n",
        "    elif desc == 'Sagittal T2/STIR':\n",
        "        trainloader, valloader, len_train, len_val = trainloader_t2stir, valloader_t2stir, len_train_t2stir, len_val_t2stir\n",
        "\n",
        "    print(f\"Training model for {desc}\")\n",
        "    train_model(model, trainloader, valloader, len_train, len_val, optimizers[desc])"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "iopub.status.busy": "2024-12-25T17:18:01.98488Z",
          "iopub.execute_input": "2024-12-25T17:18:01.985186Z",
          "execution_failed": "2024-12-26T08:22:18.133Z"
        },
        "id": "hebLmPCewQI6"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Inference"
      ],
      "metadata": {
        "id": "TD-OL24IwQI6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "train_data['level'].unique()"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.134Z"
        },
        "id": "7fUSwqHRwQI6"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "expanded_test_desc.head(5)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.134Z"
        },
        "id": "UiMsySUHwQI6"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "levels = ['l1_l2', 'l2_l3', 'l3_l4', 'l4_l5', 'l5_s1']\n",
        "\n",
        "# Function to update row_id with levels\n",
        "def update_row_id(row, levels):\n",
        "    level = levels[row.name % len(levels)]\n",
        "    return f\"{row['study_id']}_{row['condition']}_{level}\"\n",
        "\n",
        "# Update row_id in expanded_test_desc to include levels\n",
        "expanded_test_desc['row_id'] = expanded_test_desc.apply(lambda row: update_row_id(row, levels), axis=1)\n"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.135Z"
        },
        "id": "wEd9-pxSwQI6"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "expanded_test_desc.head(2)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.135Z"
        },
        "id": "IZNp4BeVwQI7"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Define a custom test dataset class\n",
        "class TestDataset(Dataset):\n",
        "    def __init__(self, dataframe, transform=None):\n",
        "        self.dataframe = dataframe\n",
        "        self.transform = transform\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.dataframe)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        image_path = self.dataframe['image_path'][index]\n",
        "        image = load_dicom(image_path)  # Define this function to load your DICOM images\n",
        "        if self.transform:\n",
        "            image = self.transform(image)\n",
        "        return image\n",
        "\n",
        "# Define the transforms\n",
        "transform = transforms.Compose([\n",
        "    transforms.ToPILImage(),\n",
        "    transforms.Resize((224, 224)),\n",
        "    transforms.Grayscale(num_output_channels=3),\n",
        "    transforms.ToTensor(),\n",
        "])\n",
        "\n",
        "# Create a test dataset and dataloader\n",
        "test_dataset = TestDataset(expanded_test_desc, transform)\n",
        "testloader = DataLoader(test_dataset, batch_size=1, shuffle=False)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.135Z"
        },
        "id": "1jefEz5gwQI7"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "for image in testloader:\n",
        "    print(image.shape)\n",
        "    break"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.135Z"
        },
        "id": "qWMHHcYVwQI7"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Function to get the model based on series_description\n",
        "def get_model(series_description):\n",
        "    return models.get(series_description, None)\n",
        "\n",
        "# Function to make predictions on the test data\n",
        "def predict_test_data(testloader, expanded_test_desc):\n",
        "    predictions = []\n",
        "    normal_mild_probs = []\n",
        "    moderate_probs = []\n",
        "    severe_probs = []\n",
        "\n",
        "    for model in models.values():\n",
        "        model.eval()\n",
        "\n",
        "    with torch.no_grad():\n",
        "        for idx, images in enumerate(tqdm(testloader)):\n",
        "            images = images.to(device)\n",
        "            series_description = expanded_test_desc.iloc[idx]['series_description']\n",
        "            model = get_model(series_description)\n",
        "            if model:\n",
        "                model.eval()  # Set the model to eval mode\n",
        "                outputs = model(images)\n",
        "                probs = torch.softmax(outputs, dim=1).squeeze(0)\n",
        "                normal_mild_probs.append(probs[0].item())\n",
        "                moderate_probs.append(probs[1].item())\n",
        "                severe_probs.append(probs[2].item())\n",
        "                predictions.append(probs)\n",
        "            else:\n",
        "                normal_mild_probs.append(None)\n",
        "                moderate_probs.append(None)\n",
        "                severe_probs.append(None)\n",
        "                predictions.append(None)\n",
        "    return normal_mild_probs, moderate_probs, severe_probs, predictions"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.135Z"
        },
        "id": "i1DNKWBWwQI7"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Make predictions on the test data\n",
        "normal_mild_probs, moderate_probs, severe_probs, test_predictions = predict_test_data(testloader, expanded_test_desc)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.136Z"
        },
        "id": "HqQx9Lc5wQI7"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "test_predictions[0]"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.136Z"
        },
        "id": "gR9LZ_BawQI7"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Add predictions and probabilities to the test DataFrame\n",
        "expanded_test_desc['normal_mild'] = normal_mild_probs\n",
        "expanded_test_desc['moderate'] = moderate_probs\n",
        "expanded_test_desc['severe'] = severe_probs"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.136Z"
        },
        "id": "OuvdUow3wQI8"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "submission = expanded_test_desc[[\"row_id\",\"normal_mild\",\"moderate\",\"severe\"]]"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.136Z"
        },
        "id": "NsES-AJ1wQI8"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "submission.head(10)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.136Z"
        },
        "id": "BVeMTEczwQI8"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "# Group by 'row_id' and sum the values\n",
        "grouped_submission = submission.groupby('row_id').max().reset_index()\n",
        "\n",
        "# Normalize the columns\n",
        "#grouped_submission[['normal_mild', 'moderate', 'severe']] = grouped_submission[['normal_mild', 'moderate', 'severe']].div(grouped_submission[['normal_mild', 'moderate', 'severe']].sum(axis=1), axis=0)\n",
        "\n",
        "# Check the first 3 rows\n",
        "grouped_submission"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.137Z"
        },
        "id": "rAvatENQwQI8"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "len(grouped_submission)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.137Z"
        },
        "id": "TKuo53mCwQI8"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "sub[['normal_mild', 'moderate', 'severe']] = grouped_submission[['normal_mild', 'moderate', 'severe']]"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.138Z"
        },
        "id": "vi901vNBwQI8"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "\n",
        "# Save the DataFrame to \"submission.csv\" in the desired directory\n",
        "sub.to_csv(\"/kaggle/working/submission.csv\", index=False)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.138Z"
        },
        "id": "8PlVQ4ukwQI9"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "sub.head(5)"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.139Z"
        },
        "id": "SLTELqNtwQI9"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "import joblib"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.139Z"
        },
        "id": "9bWm4aumwQI9"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "model"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.139Z"
        },
        "id": "QeLH1mFIwQI9"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [
        "joblib.dump(model, 'Vgg19.joblib')"
      ],
      "metadata": {
        "trusted": true,
        "execution": {
          "execution_failed": "2024-12-26T08:22:18.139Z"
        },
        "id": "ZOp583QVwQI-"
      },
      "outputs": [],
      "execution_count": null
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "trusted": true,
        "id": "G5QcTQcnwQI-"
      },
      "outputs": [],
      "execution_count": null
    }
  ]
}