example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%capture\n",
    "! pip install autoembedder\n",
    "! pip install ipywidgets==8.0.2\n",
    "! pip install plotly==5.11.0\n",
    "! pip install scikit-learn==1.2.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "from sklearn.decomposition import PCA\n",
    "from numpy.typing import NDArray\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from autoembedder import Autoembedder, dataloader, fit\n",
    "import plotly.offline as py\n",
    "import plotly.graph_objs as go\n",
    "import plotly.io as pio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pio.templates.default = \"none\"\n",
    "py.init_notebook_mode(connected=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set `data_path`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A good dataset to test the autoencoder for outlier detection is the [`Credit Card Fraud Detection`](https://www.kaggle.com/datasets/mlg-ulb/creditcardfraud) dataset from [Kaggle](https://www.kaggle.com/). To use it in the notebook please download the dataset and set the `data_path` variable to the path of the downloaded data. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = \"path/to/your/data.csv\"  # Path to your data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `plot_scatter`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_scatter(X: NDArray, y: NDArray) -> None:\n",
    "    X = PCA(n_components=2, random_state=42).fit_transform(X)\n",
    "    traces = [\n",
    "        go.Scatter(\n",
    "            x=X[y == 0, 0],\n",
    "            y=X[y == 0, 1],\n",
    "            mode=\"markers\",\n",
    "            showlegend=True,\n",
    "            name=\"Non Fraud (0)\",\n",
    "        ),\n",
    "        go.Scatter(\n",
    "            x=X[y == 1, 0],\n",
    "            y=X[y == 1, 1],\n",
    "            mode=\"markers\",\n",
    "            showlegend=True,\n",
    "            name=\"Fraud (1)\",\n",
    "        ),\n",
    "    ]\n",
    "    py.iplot(go.Figure(data=traces))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data overview"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read CSV and scale `Time` column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(data_path)\n",
    "df[\"Time\"] = df[\"Time\"] / 3600 % 24"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get sample DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = (\n",
    "    pd.concat([df.loc[df[\"Class\"] == 1], df.loc[df[\"Class\"] == 0]])\n",
    "    .sample(frac=1)\n",
    "    .reset_index(drop=True)\n",
    ")\n",
    "y = df.pop(\"Class\")\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "df = scaler.fit_transform(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_scatter(df, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Autoembedder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read data and split by target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(data_path)\n",
    "train_df = df.sample(frac=0.8)\n",
    "test_df = df.drop(train_df.index)\n",
    "\n",
    "X_train_df = train_df.query(\"Class==0\").drop(\"Class\", axis=1)\n",
    "X_test_df = test_df.drop(\"Class\", axis=1)\n",
    "y_test = test_df[\"Class\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Scale and prepare data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some usual preprocessing steps are applied to the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = MinMaxScaler()\n",
    "X_train = scaler.fit_transform(X_train_df)\n",
    "X_test = scaler.transform(X_test_df)\n",
    "\n",
    "X_train_df = pd.DataFrame(X_train, columns=X_train_df.columns)\n",
    "X_test_df = pd.DataFrame(X_test, columns=X_test_df.columns)\n",
    "eval_df = pd.DataFrame(\n",
    "    np.concatenate((X_test, y_test.to_numpy()[:, None]), axis=1), columns=df.columns\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create data loaders"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, we create two [`dataloaders`](https://chrislemke.github.io/autoembedder/autoembedder.data/#autoembedder.data.Dataset.__init__). One for training, and the other for validation data. As `source` they either accept a path to a Parquet file, to a folder of Parquet files or a [Pandas](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html)/[Dask](https://docs.dask.org/en/stable/dataframe.html) DataFrame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dl = dataloader(X_train_df)\n",
    "test_dl = dataloader(X_test_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we need to set the parameters. They are going to be used for handling the data and training the model. In this example, only parameters for the training are set. [Here](https://chrislemke.github.io/autoembedder/#parameters) you find a list of all possible parameters. This should do it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parameters = {\n",
    "    \"hidden_layers\": [[30, 25], [25, 20], [20, 15], [15, 8]],\n",
    "    \"epochs\": 10,\n",
    "    \"lr\": 0.0001,\n",
    "    \"verbose\": 1,\n",
    "    \"target\": \"Class\",\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create instance of the autoencoder model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then, we need to initialize the [autoembedder](https://chrislemke.github.io/autoembedder/autoembedder.model/#autoembedder.model.Autoembedder). In this example, we are not using any categorical features. So we can skip the `embedding_sizes` argument."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Autoembedder(parameters, num_cont_features=X_train_df.shape[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fit autoencoder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Everything is set up. Now we can [fit](https://chrislemke.github.io/autoembedder/autoembedder.learner/#autoembedder.learner.fit) the model. Since we are also passing data for [evaluation](https://chrislemke.github.io/autoembedder/autoembedder.evaluator/#autoembedder.evaluator.loss_delta) (`eval_df`), after the model is fitted it will be evaluated. \n",
    "`mean loss delta` and `median loss delta` express the differences between the mean/median loss of the `Class` `0` and `Class` `1` samples. A higher value indicates that the model is able to distinguish between the two classes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fit(parameters, model, train_dl, test_dl, eval_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict test data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We use the trained encoder form the model to predict the evaluation data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_non_fraud = eval_df.query(\"Class == 0\").drop(\"Class\", axis=1).to_numpy()\n",
    "X_fraud = eval_df.query(\"Class == 1\").drop(\"Class\", axis=1).to_numpy()\n",
    "\n",
    "with torch.no_grad():\n",
    "    model.eval()\n",
    "    non_fraud_encoded = model.encoder(torch.from_numpy(X_non_fraud))\n",
    "    fraud_encoded = model.encoder(torch.from_numpy(X_fraud))\n",
    "encoded_X = np.append(non_fraud_encoded[:5000], fraud_encoded, axis=0)\n",
    "encoded_y = np.append(np.zeros(5000), np.ones(len(fraud_encoded)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot results from test data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To get an overview how the autoembedder performed we plot the data once more."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_scatter(encoded_X, encoded_y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "autoembedder",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12 | packaged by conda-forge | (main, Mar 24 2022, 23:25:14) \n[Clang 12.0.1 ]"
  },
  "vscode": {
   "interpreter": {
    "hash": "950ba396633c7210a34f742c6b1745f808a43c09714e8882a4536677dfcbb43b"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}