Autoencoder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "from torch import nn, optim\n",
    "import numpy as np\n",
    "from random import randint\n",
    "import copy                                                # to keep track of the evolutoin of the weights of the network\n",
    "import os   \n",
    "# from sklearn import manifold                               # for the t-SNE\n",
    "# import matplotlib                                          # for the t-SNE\n",
    "# from itertools import cycle, islice                        # for the t-SNE\n",
    "from matplotlib import pyplot as plt \n",
    "from collections import OrderedDict\n",
    "import pandas as pd\n",
    "import datetime\n",
    "import time\n",
    "import torch                        \n",
    "import torch.utils.data\n",
    "from torch.nn import functional as F\n",
    "# from torchvision import transforms\n",
    "from datasets import YeastDataset # dataloader loading the yeast dataset in the correct format, replace it with one suitable for your problem\n",
    "from data_elaboration_utilities import *\n",
    "print(\"Libraries loaded\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the arrays, the dictionary and the variables\n",
    "repetitions = 50                                                              # number of random attempts\n",
    "\n",
    "parameters = {\"batch_size\" : None, \n",
    "              \"epochs\" : None,                          \n",
    "              \"learning_rate\" : None,\n",
    "              \"dataset_path\" : None,\n",
    "              \"training_path\" : None,\n",
    "              \"test_path\" : None,\n",
    "              # \"num_workers\" : 1, \n",
    "              # \"pin_memory\" : True,      \n",
    "              \"num_neurons\" : None,\n",
    "              \"z_size\" : None,\n",
    "              \"optimizer\" : None,\n",
    "              \"dropout\" : None,\n",
    "              \"reconstruction_loss\" : None,        # reconstruction losses to use\n",
    "              \"reconstruction_weight\" : None,    # weight of the reconstruction loss   # default  input_size\n",
    "              \"KLD_weight\" : None,             # weight of the KLD loss      # default beta*z_size\n",
    "              \"beta\" : None, \n",
    "              \"gamma\" : 0.1,\n",
    "              \"validation_loss\" : None,\n",
    "              \"training_loss\" : None,\n",
    "              \"KLD_loss\" : None,                    # validation\n",
    "              \"reconstruction_loss_evol\" : None,    # value of the reconstruction loss (validation)\n",
    "              \"experiment\": None}\n",
    "\n",
    "# batch_size = [32, 64, 128]            # same for training validation and test set, values for first part of the training\n",
    "\n",
    "batch_size = {\"fluxes\" : [32, 64, 128], \"genes\" : [32, 64, 128], \"all\" : [32, 64, 128], \n",
    "              \"RFECV_genes\" : [32, 64, 128], \"RFECV_fluxes\" : [32, 64, 128], \"RFECV_all\" : [32, 64, 128],  \n",
    "              \"FCBF_fluxes\" : [32, 64, 128], \"FCBF_genes\" : [32, 64, 128], \"FCBF_all\" : [32, 64, 128]}\n",
    "\n",
    "# epochs = [400, 800, 1200, 1600, 2000, 2400]      # values for first part of the training\n",
    "\n",
    "epochs = {\"fluxes\" : [400, 800, 1200, 1600, 2000, 2400], \"genes\" : [400, 800, 1200, 1600, 2000, 2400], \n",
    "          \"all\" : [400, 800, 1200, 1600, 2000, 2400], \"RFECV_genes\" : [400, 800, 1200, 1600, 2000, 2400], \n",
    "          \"RFECV_fluxes\" : [400, 800, 1200, 1600, 2000, 2400], \"RFECV_all\" : [400, 800, 1200, 1600, 2000, 2400], \n",
    "          \"FCBF_fluxes\" : [400, 800, 1200, 1600, 2000, 2400], \"FCBF_genes\" : [400, 800, 1200, 1600, 2000, 2400], \n",
    "          \"FCBF_all\" : [400, 800, 1200, 1600, 2000, 2400]}\n",
    "\n",
    "# learning_rate =  [1e-2, 1e-3, 1e-4, 1e-5]        # values for first part of the training\n",
    "\n",
    "learning_rate = {\"fluxes\" : [1e-2, 1e-3, 1e-4, 1e-5], \"genes\" : [1e-2, 1e-3, 1e-4, 1e-5], \"all\" : [1e-2, 1e-3, 1e-4, 1e-5], \n",
    "                 \"RFECV_genes\" : [1e-2, 1e-3, 1e-4, 1e-5], \"RFECV_fluxes\" : [1e-2, 1e-3, 1e-4, 1e-5], \n",
    "                 \"RFECV_all\" : [1e-2, 1e-3, 1e-4, 1e-5], \"FCBF_fluxes\" : [1e-2, 1e-3, 1e-4, 1e-5], \n",
    "                 \"FCBF_genes\" : [1e-2, 1e-3, 1e-4, 1e-5], \"FCBF_all\" : [1e-2, 1e-3, 1e-4, 1e-5]}\n",
    "\n",
    "# dropout = [0, 0.3, 0.6]                          # values for first part of the training\n",
    "\n",
    "dropout = {\"fluxes\" : [0, 0.3, 0.6], \"genes\" : [0, 0.3, 0.6], \"all\" : [0, 0.3, 0.6], \n",
    "           \"RFECV_genes\" : [0, 0.3, 0.6], \"RFECV_fluxes\" : [0, 0.3, 0.6], \"RFECV_all\" : [0, 0.3, 0.6], \n",
    "           \"FCBF_fluxes\" : [0, 0.3, 0.6], \"FCBF_genes\" : [0, 0.3, 0.6], \"FCBF_all\" : [0, 0.3, 0.6]}\n",
    "\n",
    "beta = {\"fluxes\" : [3, 4, 5], \"genes\" : [3, 4, 5], \"all\" : [3, 4, 5], \"RFECV_fluxes\" : [3, 4, 5], \n",
    "        \"RFECV_genes\" : [3, 4, 5], \"RFECV_all\" : [3, 4, 5], \"FCBF_fluxes\" : [3, 4, 5], \"FCBF_genes\" : [3, 4, 5], \n",
    "        \"FCBF_all\" : [3, 4, 5]}\n",
    "\n",
    "# optimizer = [\"Adam\", \"SGD\",\"Rprop\", \"Adadelta\"]  # values for first part of the training\n",
    "\n",
    "optimizer = {\"fluxes\" : [\"Adam\", \"SGD\",\"Rprop\", \"Adadelta\"], \"genes\" : [\"Adam\", \"SGD\",\"Rprop\", \"Adadelta\"], \n",
    "             \"all\" : [\"Adam\", \"SGD\",\"Rprop\", \"Adadelta\"], \"RFECV_genes\" : [\"Adam\", \"SGD\",\"Rprop\", \"Adadelta\"], \n",
    "             \"RFECV_fluxes\" : [\"Adam\", \"SGD\",\"Rprop\", \"Adadelta\"], \"RFECV_all\" : [\"Adam\", \"SGD\",\"Rprop\", \"Adadelta\"], \n",
    "             \"FCBF_fluxes\" : [\"Adam\", \"SGD\",\"Rprop\", \"Adadelta\"], \"FCBF_genes\" : [\"Adam\", \"SGD\",\"Rprop\", \"Adadelta\"], \n",
    "             \"FCBF_all\" : [\"Adam\", \"SGD\",\"Rprop\", \"Adadelta\"]}\n",
    "\n",
    "# loss = [\"L1_loss\", \"MSE_loss\", \"Smooth_l1_loss\"]     # values for first part of the training\n",
    "\n",
    "reconstruction_loss = {\"fluxes\" : [\"L1_loss\", \"MSE_loss\", \"Smooth_l1_loss\"], \"genes\" : [\"L1_loss\", \"MSE_loss\", \"Smooth_l1_loss\"],\n",
    "                       \"all\" : [\"L1_loss\", \"MSE_loss\", \"Smooth_l1_loss\"], \"RFECV_fluxes\" : [\"L1_loss\", \"MSE_loss\", \"Smooth_l1_loss\"],\n",
    "                       \"RFECV_genes\" : [\"L1_loss\", \"MSE_loss\", \"Smooth_l1_loss\"], \"RFECV_all\" : [\"L1_loss\", \"MSE_loss\", \"Smooth_l1_loss\"], \n",
    "                       \"FCBF_fluxes\" : [\"L1_loss\", \"MSE_loss\", \"Smooth_l1_loss\"], \"FCBF_genes\" : [\"L1_loss\", \"MSE_loss\", \"Smooth_l1_loss\"],\n",
    "                       \"FCBF_all\" : [\"L1_loss\", \"MSE_loss\", \"Smooth_l1_loss\"]}\n",
    "\n",
    "\n",
    "paths = [\"path\", None       # to distinguish between the two possible situations in which there is one or two datasets (not training, validation and test together)\n",
    "        ]   # add the paths to the csv containing your data according to the usage of the variable paths below\n",
    "\n",
    "\n",
    "no_cuda = False               # SHOULD BE FALSE                    \n",
    "seed = 1                   \n",
    "log_interval = 10       \n",
    "percent_train = 0.7  \n",
    "percent_validation = 0.2\n",
    "\n",
    "dataset_path = paths[3] \n",
    "parameters[\"dataset_path\"] = dataset_path\n",
    "\n",
    "training_path = paths[-1]                                                      \n",
    "test_path = paths[-1] \n",
    "parameters[\"training_path\"] = training_path\n",
    "parameters[\"test_path\"] = test_path\n",
    "                                               \n",
    "\n",
    "\n",
    "input_size = \n",
    "\n",
    "reconstruction_weight = [input_size]          # default\n",
    "\n",
    "# change the number of neurons to search according to the same of your data\n",
    "# the keys of the dictionary correspond to different types of input data\n",
    "num_neurons = {\"fluxes\" : [459, 520, 330, 200], \"genes\" : [6170, 6800, 3500, 2500], \"all\" : [6629, 7200, 4000, 2600], \n",
    "               \"RFECV_genes\" : [1858, 2350, 750, 550], \"RFECV_all\" : [1664, 2300, 600, 440], \"RFECV_fluxes\" : [19, 25, 7, 5], \n",
    "               \"FCBF_genes\" : [79, 90, 55, 40], \"FCBF_fluxes\" : [12, 18, 6, 4],\"FCBF_all\" : [68, 80, 46, 32]}  \n",
    "\n",
    "# same as above\n",
    "z_size = {\"fluxes\" : [200, 130, 80, 40], \"genes\" : [2500, 1200, 800, 500], \"all\" : [2600, 1300, 900, 600], \n",
    "          \"RFECV_genes\" : [550, 400, 320, 200], \"RFECV_all\" : [440, 280, 200, 170], \"RFECV_fluxes\" : [5, 4, 3, 2],\n",
    "          \"FCBF_genes\" : [40, 32, 24, 15], \"FCBF_fluxes\" : [4, 3, 2, 1], \"FCBF_all\" : [32, 20, 16, 8]} \n",
    "\n",
    "# defines the mapping according to the input size\n",
    "mapping = {459 : \"fluxes\", 6170 : \"genes\", 6629 : \"all\", 1858 : \"RFECV_genes\", 19 : \"RFECV_fluxes\", 1664 : \"RFECV_all\", \n",
    "           79 : \"FCBF_genes\", 12 : \"FCBF_fluxes\", 68 : \"FCBF_all\"}\n",
    "\n",
    "num_n = mapping[input_size]\n",
    "\n",
    "cuda = not no_cuda and torch.cuda.is_available()\n",
    "torch.manual_seed(seed)\n",
    "\n",
    "device = torch.device(\"cuda\" if cuda else \"cpu\")              # launch on the GPU if available\n",
    "\n",
    "kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}\n",
    "\n",
    "if dataset_path != None:\n",
    "    partition, labels = splitTrainingValidationTestSet(dataset_path, percent_train, percent_validation)\n",
    "else:\n",
    "    partition, labels = loadTrainingValidationTestSet(training_path, test_path, percent_train, percent_validation)\n",
    "    \n",
    "training_set = YeastDataset(partition['training'], labels)\n",
    "validation_set = YeastDataset(partition['validation'], labels)\n",
    "        \n",
    "torch.set_printoptions(precision=9)                                 # to print more digits for the loss\n",
    "\n",
    "print(\"Device: \", device)                           # simple check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dataset_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def RandomIndexes():                                     # randomly choose the parameter\n",
    "    int1 = randint(0, len(batch_size[num_n]) - 1)    \n",
    "    int2 = randint(0, len(epochs[num_n]) - 1)\n",
    "    int3 = randint(0, len(learning_rate[num_n]) - 1)\n",
    "    int4 = randint(0, len(num_neurons[num_n]) - 1)                                  \n",
    "    int5 = randint(0, len(z_size[num_n]) - 1)\n",
    "    int6 = randint(0, len(optimizer[num_n]) - 1)\n",
    "    int7 = randint(0, len(dropout[num_n]) - 1)\n",
    "    int8 = randint(0, len(reconstruction_loss[num_n]) - 1) \n",
    "    int9 = randint(0, len(beta[num_n]) - 1) \n",
    "    return int1, int2, int3, int4, int5, int6, int7, int8, int9\n",
    "\n",
    "def fillDict(d):\n",
    "    ind1, ind2, ind3, ind4, ind5, ind6, ind7, ind8, ind9 = RandomIndexes()\n",
    "    d[\"batch_size\"] = batch_size[num_n][ind1]\n",
    "    d[\"epochs\"] = epochs[num_n][ind2]\n",
    "    d[\"learning_rate\"] = learning_rate[num_n][ind3]\n",
    "    d[\"num_neurons\"] = num_neurons[num_n][ind4]                          \n",
    "    d[\"z_size\"] = z_size[num_n][ind5]\n",
    "    d[\"optimizer\"] = optimizer[num_n][ind6]\n",
    "    d[\"dropout\"] = dropout[num_n][ind7]\n",
    "    d[\"reconstruction_loss\"] = reconstruction_loss[num_n][ind8]\n",
    "    d[\"beta\"] = beta[num_n][ind9]\n",
    "    d = OrderedDict(sorted(d.items(), key=lambda t: t[0]))\n",
    "    return d                                                                       \n",
    "\n",
    "def checkIdentical(path, parameters):                        \n",
    "    combinations = pd.read_csv(path) \n",
    "    if \"Unnamed: 0\" in combinations.columns:\n",
    "        combinations.drop(\"Unnamed: 0\", axis=1, inplace=True)        # change according to the structure of your file\n",
    "    combinations = combinations.to_dict('records')\n",
    "    if parameters in combinations:\n",
    "        print(\"This combination of parameters has already been used\")\n",
    "        return (False, None)                                                        # return false if combination already used\n",
    "    print(parameters)\n",
    "    return (True, parameters)                                                       # true if combination not used yet\n",
    "\n",
    "def checkForWarmStarting(path, num1, num2):                                     # returns False if no warmstarting is possible\n",
    "    df = pd.read_csv(path)\n",
    "    array = [num2]\n",
    "    df = df.loc[(df['num_neurons'] == num1) & df['z_size'].isin(array)]\n",
    "    if len(df) != 0:\n",
    "        min_val = sorted(df[\"validation_loss\"])                                # returns the one with the best validation loss\n",
    "        exp = df.loc[df['validation_loss'] == min_val[0]][\"experiment\"]\n",
    "        return exp.iloc[0]\n",
    "    return False  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class trialVAE(nn.Module):\n",
    "    def __init__(self, num_neurons, z_size, dropout):\n",
    "        super(trialVAE, self).__init__()\n",
    "    \n",
    "        self.input_size = input_size\n",
    "        self.num_neurons = num_neurons\n",
    "        self.z_size = z_size\n",
    "        self.dropout = dropout\n",
    "        \n",
    "        self.fc1 = nn.Linear(self.input_size, self.num_neurons)\n",
    "        self.fc21 = nn.Linear(self.num_neurons, self.z_size)\n",
    "        self.fc22 = nn.Linear(self.num_neurons, self.z_size)\n",
    "        self.fc3 = nn.Linear(self.z_size, self.num_neurons)\n",
    "        self.fc4 = nn.Linear(self.num_neurons, self.input_size)\n",
    "        self.dropt = nn.Dropout(self.dropout)\n",
    "\n",
    "    def encode(self, x):\n",
    "        h1 = F.relu(self.fc1(x.float()))              \n",
    "        # h1 = torch.tanh(self.fc1(x.float()))    # MSE doesn't learn and all the weights change at approximately each epoch\n",
    "        return self.fc21(self.dropt(h1)), self.fc22(self.dropt(h1))\n",
    "\n",
    "    def reparameterize(self, mu, logvar):\n",
    "        std = torch.exp(0.5*logvar)\n",
    "        eps = torch.randn_like(std)\n",
    "        return eps.mul(std).add_(mu)\n",
    "\n",
    "    def decode(self, z):\n",
    "        h3 = F.relu(self.fc3(z))\n",
    "        # h3 = torch.tanh(self.fc3(z))               # MSE doesn't learn and all the weights change at approximately each epoch\n",
    "        return torch.sigmoid(self.fc4(self.dropt(h3)))         \n",
    "        # return F.softmax(self.fc4(h3), dim=1)      # MSE doesn't decrease with it\n",
    "        # return torch.tanh(self.fc4(h3))            # MSE doesn't decrease like with the sigmoid\n",
    "\n",
    "    def forward(self, x):\n",
    "        mu, logvar = self.encode(x)\n",
    "        z = self.reparameterize(mu, logvar)        # representation to get\n",
    "        return self.decode(z), mu, logvar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_function(real, predict, loss, mu, logvar, epoch, tot_epoch):\n",
    "    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())\n",
    "    if loss == \"L1_loss\":\n",
    "        return torch.mean(reconstruction_weight[0]*F.l1_loss(predict, real) + KLD_weight*KLD), reconstruction_weight[0]*F.l1_loss(predict, real), KLD_weight*KLD\n",
    "    elif loss == \"MSE_loss\":\n",
    "        return torch.mean(reconstruction_weight[0]*F.mse_loss(predict, real) + KLD_weight*KLD), reconstruction_weight[0]*F.mse_loss(predict, real), KLD_weight*KLD\n",
    "    else:\n",
    "        return torch.mean(reconstruction_weight[0]*F.smooth_l1_loss(predict, real) + KLD_weight*KLD), reconstruction_weight[0]*F.smooth_l1_loss(predict, real), KLD_weight*KLD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimizer_function(optimizer, model, lr):\n",
    "    if optimizer == \"Adam\":\n",
    "        return optim.Adam(model.parameters(), lr=lr)\n",
    "    elif optimizer == \"SGD\":\n",
    "        return optim.SGD(model.parameters(), lr=lr, weight_decay=0.1)\n",
    "    elif optimizer == \"RMSprop\":\n",
    "        return optim.RMSprop(model.parameters(), lr=lr)\n",
    "    else:\n",
    "        return optim.Adadelta(model.parameters(), lr=lr)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch, lo, tot_epoch):   \n",
    "    model.train()\n",
    "    train_loss = 0\n",
    "    for _, (data, labels) in enumerate(train_loader):\n",
    "        # print(\"Epoch: {}, Batch index: {}\".format(epoch, batch_idx))\n",
    "        data = data.to(device)\n",
    "        optmz.zero_grad()\n",
    "        recon_batch, mu, logvar = model(data)\n",
    "        loss, reconstruction_loss, KLD = loss_function(data.float(), recon_batch, lo, mu, logvar, epoch, tot_epoch)\n",
    "        # assert((reconstruction_loss == reconstruction_loss).item() == 1)  # check whether we're going to get NaNs\n",
    "        loss.backward()\n",
    "        train_loss += loss.item()\n",
    "        optmz.step()\n",
    "    with torch.no_grad():\n",
    "        loss_evolution.append(loss.cpu().detach().numpy())   \n",
    "        KLD_evolution.append(KLD.cpu().detach().numpy()) \n",
    "        reconstruction_loss_evolution.append(reconstruction_loss.cpu().detach().numpy()) \n",
    "        \n",
    "    print('====> Epoch: {} Average loss: {:.4f}'.format(          \n",
    "        epoch, loss / len(train_loader.dataset)))\n",
    "\n",
    "    # print('====> Epoch: {} Average loss: {:.4f}'.format(\n",
    "    #      epoch, train_loss / len(train_loader.dataset)))\n",
    "\n",
    "\n",
    "def validate(epoch, lo, tot_epoch):\n",
    "    model.eval()\n",
    "    valid_loss = 0\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, (data, labels) in enumerate(validation_loader):\n",
    "            data = data.to(device)\n",
    "            recon_batch, mu, logvar = model(data)\n",
    "            loss, reconstruction_loss, KLD = loss_function(data.float(), recon_batch, lo, mu, logvar, epoch, tot_epoch)\n",
    "            valid_loss += loss.item()\n",
    "        valid_loss_evolution.append(loss.cpu().numpy()) \n",
    "        valid_KLD.append(KLD.cpu().numpy()) \n",
    "        valid_reconstruction.append(reconstruction_loss.cpu().numpy()) \n",
    "    # valid_loss /= len(validation_loader.dataset)                              \n",
    "    # print('====> Validation set loss: {:.4f}'.format(valid_loss))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(repetitions):\n",
    "    while(True):\n",
    "        flag, parameters = checkIdentical(\"path of file containing all the previously used combinations\\Records.csv\", fillDict(parameters))\n",
    "        if flag:                                                              # if combination never used exit\n",
    "            break\n",
    "    directory_path = \"path to the directory where to save the results\" + \"\\\\\" + str(datetime.datetime.now().\n",
    "                                                                                     strftime(\"%d_%m_%Y-%H.%M\"))\n",
    "    \n",
    "    \n",
    "    print(\"Repetition: \", i + 1)\n",
    "    model = trialVAE(parameters[\"num_neurons\"], parameters[\"z_size\"], parameters[\"dropout\"])\n",
    "                                        # check for a possible warm starting\n",
    "    check = checkForWarmStarting(\"path\", # to the file containing all the tried combinations, different from the one above in case you want\n",
    "                                 parameters[\"num_neurons\"], parameters[\"z_size\"])# to initialise the network with pre-computed weights\n",
    "    if check :\n",
    "        print(\"Warm starting from\", check)\n",
    "        model.load_state_dict(torch.load(\"path to the associated weights\"), \n",
    "                                         strict=True)\n",
    "    \n",
    "    model.to(device)\n",
    "    \n",
    "    optmz = optimizer_function(parameters[\"optimizer\"], model, parameters[\"learning_rate\"])\n",
    "    step_size = parameters[\"epochs\"]//2\n",
    "    gamma = parameters[\"gamma\"]\n",
    "    scheduler = optim.lr_scheduler.StepLR(optmz, step_size=step_size, gamma=gamma)\n",
    "    \n",
    "    KLD_weight = parameters[\"beta\"]*parameters[\"z_size\"]          # default\n",
    "    \n",
    "    train_loader = torch.utils.data.DataLoader(training_set, shuffle=True, batch_size=parameters[\"batch_size\"], **kwargs)\n",
    "    validation_loader = torch.utils.data.DataLoader(validation_set, batch_size=parameters[\"batch_size\"], shuffle=True, **kwargs)\n",
    "    \n",
    "    loss_evolution = list()                                                 # training losses\n",
    "    KLD_evolution = list()\n",
    "    reconstruction_loss_evolution = list()\n",
    "    \n",
    "    valid_loss_evolution = list()                                           # validation losses\n",
    "    valid_KLD = list()\n",
    "    valid_reconstruction = list()\n",
    "    \n",
    "    # training\n",
    "    for epoch in range(1, parameters[\"epochs\"] + 1):                                   \n",
    "        scheduler.step()\n",
    "        train(epoch, parameters[\"reconstruction_loss\"], parameters[\"epochs\"])\n",
    "        validate(epoch, parameters[\"reconstruction_loss\"], parameters[\"epochs\"])  \n",
    "        \n",
    "    # save the model    \n",
    "    if not os.path.exists(directory_path):                \n",
    "        os.makedirs(directory_path)\n",
    "    \n",
    "    #save the losses for comparison (superfluous given the saves in the csv file)\n",
    "    text_file = open(\"path\\Losses.txt\", \"a\")\n",
    "    text_file.write(\"\\n\" + directory_path[68:] + \"  \" + str(valid_loss_evolution[-10:])) # change according to your directory_path\n",
    "    text_file.close()\n",
    "    \n",
    "    torch.save(model.state_dict(), directory_path + \"\\\\weights.pt\")   \n",
    "    \n",
    "    visualizeLossesOverEpochs(valid_loss_evolution, loss_evolution, 0,       # comparison between validation and training losses\n",
    "                          \"Validation loss\", \"Training loss\", \"0\", 50, 100, 'o-', False)\n",
    "    \n",
    "    plt.savefig(fname=directory_path + \"\\\\validation_loss.png\", bbox_inches=\"tight\")\n",
    "    \n",
    "    visualizeLossesOverEpochs(loss_evolution, KLD_evolution, reconstruction_loss_evolution,\n",
    "                          \"Training loss\", \"KLD loss\", \"Reconstruction loss\", 50, 100, 'o-', False)  # comparison between training, KLD and reconstruction losses\n",
    "\n",
    "    plt.savefig(fname=directory_path + \"\\\\training_loss.png\", bbox_inches=\"tight\")\n",
    "    \n",
    "    # save full validation loss\n",
    "    text_file = open(directory_path +\"\\\\validation_loss.txt\", \"w\")\n",
    "    text_file.write(str(valid_loss_evolution) + \"\\n\")\n",
    "    text_file.close()\n",
    "    \n",
    "    # save full KLD loss (validation)\n",
    "    text_file = open(directory_path +\"\\\\KLD_loss.txt\", \"w\")\n",
    "    text_file.write(str(valid_KLD) + \"\\n\")\n",
    "    text_file.close()\n",
    "    \n",
    "    # save full reconstruction loss (validation)\n",
    "    text_file = open(directory_path +\"\\\\reconstruction_loss.txt\", \"w\")\n",
    "    text_file.write(str(valid_reconstruction) + \"\\n\")\n",
    "    text_file.close()\n",
    "    \n",
    "    # write the parameters in the csv file\n",
    "    combinations = pd.read_csv(\"path of file containing all the previously used combinations\\Records.csv\") \n",
    "    if \"Unnamed: 0\" in combinations.columns:\n",
    "        combinations.drop(\"Unnamed: 0\", axis=1, inplace=True)\n",
    "    combinations = combinations.to_dict('records')\n",
    "    parameters[\"training_loss\"] = loss_evolution[-1]\n",
    "    parameters[\"validation_loss\"] = valid_loss_evolution[-1]\n",
    "    parameters[\"KLD_loss\"] = valid_KLD[-1]                                  # N.B. these are the validation losses\n",
    "    parameters[\"KLD_weight\"] = KLD_weight\n",
    "    parameters[\"reconstruction_weight\"] = reconstruction_weight[0]\n",
    "    parameters[\"reconstruction_loss_evol\"] = valid_reconstruction[-1]\n",
    "    parameters[\"training_path\"] = str(training_path)\n",
    "    parameters[\"test_path\"] = str(test_path)\n",
    "    parameters[\"experiment\"] = directory_path[68:]\n",
    "    combinations.append(parameters)                                               \n",
    "    records = pd.DataFrame(combinations)\n",
    "    records.to_csv(\"path of file containing all the previously used combinations\\Records.csv\", encoding=\"utf-8\")\n",
    "    \n",
    "    text_file = open(directory_path + \"\\\\\" + \"Parameters.txt\", \"w\")\n",
    "    text_file.write(\"Parameters used: \\n\\n\")\n",
    "    for _, (key, value) in enumerate(parameters.items()):\n",
    "        text_file.write(key + \" = \" + str(value) + \"\\n\")\n",
    "    text_file.write(\"\\n Dataset path: \" + dataset_path)\n",
    "    text_file.write(\"\\n Training set path: \" + str(training_path))  \n",
    "    text_file.write(\"\\n Test set path: \" + str(test_path))\n",
    "    \n",
    "    text_file.close()\n",
    "    \n",
    "    plt.close(\"all\")                                            # close all the figures in order to save memory"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}