Agent32.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0oS0QrSjFeet",
    "outputId": "c039a77b-6e3f-4560-be36-962df55d506d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive',force_remount=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 35
    },
    "id": "rWiNrnhtFmlL",
    "outputId": "59340d85-728f-4d4a-f4c2-135d799fcf32"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "string"
      },
      "text/plain": [
       "'colab.zip'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import shutil,os\n",
    "shutil.copy(\"/content/drive/MyDrive/Controller32/colabM1M5.zip\", \"colab.zip\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "HRQRORdwFqu1"
   },
   "outputs": [],
   "source": [
    "import zipfile\n",
    "\n",
    "# Replace 'file.zip' with the name of your zip file\n",
    "with zipfile.ZipFile('colab.zip', 'r') as zip_ref:\n",
    "    zip_ref.extractall()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YSEEeDSBNBsL"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L5l5czSpGLpG"
   },
   "source": [
    "Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "FQOtup6WFt75",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import random, shutil, pickle, sys\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "from keras import Sequential\n",
    "from keras.optimizers import RMSprop\n",
    "from IPython.display import display\n",
    "from PIL import Image\n",
    "import pandas as pd\n",
    "from keras import Model\n",
    "import keras, os, glob\n",
    "import tensorflow as tf\n",
    "from keras.layers import Layer, Dense, Conv2D, Flatten, RepeatVector,Dropout, LayerNormalization, MultiHeadAttention, GlobalAveragePooling1D\n",
    "from keras.layers import Activation, LSTM, Bidirectional , Dropout\n",
    "from keras import layers\n",
    "# import cv2\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import tensorflow as tf\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import codecs\n",
    "import csv\n",
    "import secrets\n",
    "import sqlite3\n",
    "from IPython.display import display\n",
    "from IPython.display import clear_output\n",
    "import time\n",
    "from tabulate import tabulate\n",
    "import json\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PZOmU4t_GFur"
   },
   "source": [
    "The Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "id": "N-b6e13YGKU4",
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ForexEnv():\n",
    "    def __init__(self, m5long, m5short, m1long, m1short):\n",
    "        self.forex_m5long       = m5long\n",
    "        self.forex_m5short      = m5short\n",
    "        self.forex_m1long       = m1long\n",
    "        self.forex_m1short      = m1short\n",
    "        self.dataset_directory  = \"colabM1M5/episode2/\"\n",
    "        self.current_trade      = {}\n",
    "        self.position           = None\n",
    "        self.holding            = 0\n",
    "        self.take_profit        = 400.0\n",
    "        self.stop_loss          = 400.0\n",
    "        self.stop_price         = 0.0\n",
    "        self.trailing_loss      = 0.0\n",
    "        self.lot_size           = 0.1\n",
    "        self.pip_value          = 1.5\n",
    "        self.current_step       = 4\n",
    "        self.current_end        = len(self.forex_m1short)\n",
    "        self.account_balance    = 1000.0\n",
    "        self.balance_limit      = 950.0\n",
    "        self.profit_limit       = -50\n",
    "        self.done               = 0\n",
    "        self.target_lookahead   = 1\n",
    "        self.current_trade      = {\n",
    "                                    \"current_price\"   : 0.0,\n",
    "                                    \"entry_price\"   : 0.0,\n",
    "                                    \"stop_loss\"     : 0.0,\n",
    "                                    \"take_profit\"   : 0.0,\n",
    "                                    \"stop_price\"    : 0.0,\n",
    "                                    \"profit\"        : 0.0,\n",
    "                                    \"balance\"       : self.account_balance,\n",
    "                                    \"highest\"       : 0\n",
    "                                \n",
    "        }\n",
    "        self.account_balance_norm = MinMaxScaler(feature_range=(0, 1))\n",
    "        self.holding_norm =     MinMaxScaler(feature_range=(0, 1))\n",
    "        self.profit_norm =      MinMaxScaler(feature_range=(0, 1))\n",
    "        self.action_norm=   MinMaxScaler(feature_range=(0, 1))\n",
    "        self.total_pips = 100\n",
    "        self.init_scalers()\n",
    "        \n",
    "    def reset(self, dummy_dataset = False):\n",
    "        self.current_trade = {\n",
    "            \"entry_price\" : 5000.0,\n",
    "            \"current_price\": 5000.0,\n",
    "            \"stop_loss\":5000.0,\n",
    "            \"take_profit\" : 200.0,\n",
    "            \"stop_price\": 5000.0,\n",
    "            \"profit\": 0.0,\n",
    "            \"balance\" : self.account_balance,\n",
    "            \"highest\"       : 0\n",
    "\n",
    "        }\n",
    "        self.position = None\n",
    "        self.holding = 0\n",
    "        self.current_step = 4\n",
    "        self.account_balance = 1000.0\n",
    "        self.done = 0\n",
    "        \n",
    "        return self.get_observation(self.current_step - 1 , 0, self.get_scaled_values(\"Account Balance\", self.account_balance),0,self.holding, dummy_dataset)\n",
    "\n",
    "    def read_image(self, image_path):\n",
    "\n",
    "        return np.asarray(Image.open(self.dataset_directory+image_path))/255\n",
    "    \n",
    "\n",
    "    def step(self, action, dummy_dataset=False):\n",
    "        reward = 0\n",
    "        self.stop_price = 0.0\n",
    "        \n",
    "        if self.position == None:\n",
    "            if action == 1:\n",
    "                self.position = \"long\"\n",
    "                self.holding = 1\n",
    "                current_price = float(self.forex_m1short.iloc[self.current_step][\"Ask\"])\n",
    "                self.current_trade = {\n",
    "                    \"entry_price\": current_price,\n",
    "                    \"current_price\": current_price,\n",
    "                    \"stop_loss\": current_price - self.stop_loss,\n",
    "                    \"take_profit\": current_price + self.take_profit,\n",
    "                    \"stop_price\": current_price - self.stop_loss,\n",
    "                    \"profit\": 0.0,\n",
    "                    \"balance\": self.account_balance,\n",
    "                    \"highest\"       : 0\n",
    "                    \n",
    "                }\n",
    "                self.stop_price = 0.0\n",
    "            elif action == 2:\n",
    "                self.holding = 2\n",
    "                self.position = \"short\"\n",
    "                current_price = float(self.forex_m1short.iloc[self.current_step][\"Bid\"])\n",
    "                self.current_trade = {\n",
    "                    \"entry_price\": current_price,\n",
    "                    \"current_price\": current_price,\n",
    "                    \"stop_loss\": current_price + self.stop_loss,\n",
    "                    \"take_profit\": current_price - self.take_profit,\n",
    "                    \"stop_price\": current_price + self.stop_loss,\n",
    "                    \"profit\": 0.0,\n",
    "                    \"balance\": self.account_balance,\n",
    "                    \"highest\"       : 0\n",
    "                }\n",
    "                self.stop_price = current_price - self.take_profit\n",
    "            elif action == 0:\n",
    "\n",
    "                self.current_trade = {\n",
    "                    \"entry_price\": float(self.forex_m1short.iloc[self.current_step][\"Ask\"]),\n",
    "                    \"current_price\": float(self.forex_m1short.iloc[self.current_step][\"Ask\"]),\n",
    "                    \"stop_loss\": float(self.forex_m1short.iloc[self.current_step][\"Ask\"]),\n",
    "                    \"take_profit\": float(self.forex_m1short.iloc[self.current_step][\"Ask\"]),\n",
    "                    \"stop_price\": float(self.forex_m1short.iloc[self.current_step][\"Ask\"]),\n",
    "                    \"profit\": 0.0,\n",
    "                    \"balance\": self.account_balance,\n",
    "                    \"highest\"       : 0\n",
    "                }\n",
    "                self.holding = 0\n",
    "                self.position = None\n",
    "                reward = -0.1\n",
    "                \n",
    "        elif self.position == \"long\":\n",
    "            if action == 1:\n",
    "                current_price = float(self.forex_m1short.iloc[self.current_step][\"Bid\"])\n",
    "                self.current_trade[\"current_price\"] = current_price\n",
    "                self.current_trade[\"profit\"] = (current_price - self.current_trade[\"entry_price\"]) \n",
    "                self.current_trade[\"balance\"] = self.account_balance + self.current_trade[\"profit\"]\n",
    "                if self.current_trade[\"profit\"] > self.current_trade[\"highest\"]:\n",
    "                    self.current_trade[\"highest\"] = self.current_trade[\"profit\"]\n",
    "                try:\n",
    "                    if current_price > self.current_trade[\"entry_price\"]:\n",
    "                        reward = (current_price - self.current_trade[\"entry_price\"]) / (self.current_trade[\"take_profit\"] - self.current_trade[\"entry_price\"])\n",
    "                    elif self.current_trade[\"entry_price\"] > current_price:\n",
    "                        reward = -((self.current_trade[\"take_profit\"] - current_price )/(self.current_trade[\"take_profit\"] - self.current_trade[\"stop_loss\"]) )\n",
    "                except:\n",
    "                    reward = 0\n",
    "                ratio = (1/3) * self.current_trade[\"highest\"]\n",
    "                if self.current_trade[\"highest\"] - ratio < self.current_trade[\"profit\"]:\n",
    "                    reward = -1\n",
    "            elif action == 2:\n",
    "                self.position = None\n",
    "                self.holding = 0\n",
    "                current_price = float(self.forex_m1short.iloc[self.current_step][\"Bid\"])\n",
    "                try:              \n",
    "                    if current_price > self.current_trade[\"entry_price\"]:\n",
    "                        reward = (current_price - self.current_trade[\"entry_price\"]) / (self.current_trade[\"take_profit\"] - self.current_trade[\"entry_price\"])\n",
    "                    elif self.current_trade[\"entry_price\"] > current_price:\n",
    "                        reward = -((self.current_trade[\"take_profit\"] - current_price )/(self.current_trade[\"take_profit\"] - self.current_trade[\"stop_loss\"]) )\n",
    "                except:\n",
    "                    reward = 0                \n",
    "                self.current_trade[\"profit\"] = (current_price - self.current_trade[\"entry_price\"]) \n",
    "                self.current_trade[\"current_price\"] = current_price\n",
    "                self.current_trade[\"balance\"] = self.account_balance + self.current_trade[\"profit\"]\n",
    "                self.account_balance += self.current_trade[\"profit\"]\n",
    "                self.current_trade[\"stop_price\"] = current_price\n",
    "                self.current_trade[\"stop_loss\"] = current_price \n",
    "                self.current_trade[\"take_profit\"] = current_price  \n",
    "            elif action == 0:\n",
    "                self.position = None\n",
    "                self.holding = 0\n",
    "                current_price = float(self.forex_m1short.iloc[self.current_step][\"Bid\"])\n",
    "                try:\n",
    "                    if current_price > self.current_trade[\"entry_price\"]:\n",
    "                        reward = (current_price - self.current_trade[\"entry_price\"]) / (self.current_trade[\"take_profit\"] - self.current_trade[\"entry_price\"])\n",
    "                    elif self.current_trade[\"entry_price\"] > current_price:\n",
    "                        reward = -((self.current_trade[\"take_profit\"] - current_price )/(self.current_trade[\"take_profit\"] - self.current_trade[\"stop_loss\"]) )             \n",
    "                except:\n",
    "                    reward = 0                 \n",
    "                self.current_trade[\"current_price\"] = current_price\n",
    "                self.current_trade[\"profit\"] =(current_price - self.current_trade[\"entry_price\"]) \n",
    "                self.current_trade[\"balance\"] = self.account_balance + self.current_trade[\"profit\"]\n",
    "                self.account_balance += self.current_trade[\"profit\"]                 \n",
    "                self.current_trade[\"stop_loss\"] = current_price \n",
    "                self.current_trade[\"take_profit\"] = current_price\n",
    "                self.current_trade[\"stop_price\"] = current_price                               \n",
    "        elif self.position == \"short\":\n",
    "            if action == 1:\n",
    "                self.position = None\n",
    "                self.holding = 0\n",
    "                current_price = float(self.forex_m1short.iloc[self.current_step][\"Ask\"])\n",
    "                try:               \n",
    "                    if current_price > self.current_trade[\"entry_price\"]:\n",
    "                        reward = - (abs(current_price - self.current_trade[\"take_profit\"]) / abs(self.current_trade[\"stop_loss\"]-self.current_trade[\"take_profit\"]))\n",
    "                    elif self.current_trade[\"entry_price\"] > current_price:\n",
    "                        reward = (current_price-self.current_trade[\"entry_price\"]) / (self.current_trade[\"entry_price\"]-self.current_trade[\"stop_loss\"])\n",
    "                except:\n",
    "                    reward = 0                \n",
    "                self.current_trade[\"current_price\"] = current_price\n",
    "                self.current_trade[\"profit\"] = (self.current_trade[\"entry_price\"] - current_price) \n",
    "                self.current_trade[\"balance\"] = self.account_balance + self.current_trade[\"profit\"]\n",
    "                self.account_balance += self.current_trade[\"profit\"] \n",
    "                self.current_trade[\"stop_price\"] = current_price\n",
    "                self.current_trade[\"stop_loss\"] = current_price \n",
    "                self.current_trade[\"take_profit\"] = current_price\n",
    "\n",
    "            elif action == 2:\n",
    "                current_price = float(self.forex_m1short.iloc[self.current_step][\"Ask\"])\n",
    "                self.current_trade[\"current_price\"] = current_price\n",
    "                if self.current_trade[\"profit\"] > self.current_trade[\"highest\"]:\n",
    "                    self.current_trade[\"highest\"] = self.current_trade[\"profit\"]\n",
    "                    \n",
    "                try:\n",
    "                    if current_price > self.current_trade[\"entry_price\"]:\n",
    "                        reward = - (abs(current_price - self.current_trade[\"take_profit\"]) / abs(self.current_trade[\"stop_loss\"]-self.current_trade[\"take_profit\"]))\n",
    "                    elif self.current_trade[\"entry_price\"] > current_price:\n",
    "                        reward = (current_price-self.current_trade[\"entry_price\"]) / (self.current_trade[\"entry_price\"]-self.current_trade[\"stop_loss\"])\n",
    "                except:\n",
    "                    reward = 0\n",
    "                \n",
    "                self.current_trade[\"profit\"] = (self.current_trade[\"entry_price\"] - current_price) \n",
    "                self.current_trade[\"balance\"] = self.account_balance + self.current_trade[\"profit\"]\n",
    "                ratio = (1/3) * self.current_trade[\"highest\"]\n",
    "                if self.current_trade[\"highest\"] - ratio < self.current_trade[\"profit\"]:\n",
    "                    reward = -1\n",
    "\n",
    "            elif action == 0:\n",
    "                self.position = None\n",
    "                self.holding = 0\n",
    "                current_price = float(self.forex_m1short.iloc[self.current_step][\"Ask\"])\n",
    "                try:\n",
    "                    if current_price > self.current_trade[\"entry_price\"]:\n",
    "                        reward = - (abs(current_price - self.current_trade[\"take_profit\"]) / abs(self.current_trade[\"stop_loss\"]-self.current_trade[\"take_profit\"]))\n",
    "                    elif self.current_trade[\"entry_price\"] > current_price:\n",
    "                        reward = (current_price-self.current_trade[\"entry_price\"]) / (self.current_trade[\"entry_price\"]-self.current_trade[\"stop_loss\"])\n",
    "                except:\n",
    "                    reward = 0                \n",
    "                self.current_trade[\"current_price\"] = current_price\n",
    "                self.current_trade[\"profit\"] = (self.current_trade[\"entry_price\"] - current_price) \n",
    "                self.current_trade[\"balance\"] = self.account_balance + self.current_trade[\"profit\"]\n",
    "                self.account_balance += self.current_trade[\"profit\"] \n",
    "                self.current_trade[\"stop_loss\"] = current_price \n",
    "                self.current_trade[\"take_profit\"] = current_price\n",
    "                self.current_trade[\"stop_price\"] = current_price                \n",
    "        self.current_step += 1\n",
    "        if self.current_step >= self.current_end + 1:\n",
    "            self.done = 1\n",
    "        if  int(self.account_balance) < self.balance_limit or self.current_trade[\"profit\"]< self.profit_limit:\n",
    "            reward = -1\n",
    "            self.done = 1\n",
    "            \n",
    "        info = {\n",
    "            \"_step\": self.current_step,\n",
    "            \"c_trade\": self.current_trade,\n",
    "            \"c_price\": float(self.forex_m1short.iloc[self.current_step][\"Bid\"]) if self.position == \"long\" else float(self.forex_m1short.iloc[self.current_step][\"Ask\"])\n",
    "        }\n",
    "        if dummy_dataset:\n",
    "            next_state, long_short_dict = self.get_observation(self.current_step + self.target_lookahead, self.get_scaled_values(\"Action\", action),self.get_scaled_values(\"Account Balance\", self.current_trade[\"balance\"]), self.get_scaled_values(\"Profit\", self.current_trade[\"profit\"]),self.get_scaled_values(\"Holding\", self.holding), True)\n",
    "        else:\n",
    "            next_state,long_short_dict = self.get_observation(self.current_step + self.target_lookahead, self.get_scaled_values(\"Action\", action),self.get_scaled_values(\"Account Balance\", self.current_trade[\"balance\"]), self.get_scaled_values(\"Profit\", self.current_trade[\"profit\"]),self.get_scaled_values(\"Holding\", self.holding), False)\n",
    "        \n",
    "        long_short_dict[\"reward\"] = reward\n",
    "        long_short_dict[\"action\"] = self.get_scaled_values(\"Action\", action)\n",
    "        long_short_dict[\"done\"] = self.done\n",
    "        long_short_dict[\"info\"] = info\n",
    "        long_short_dict[\"account_balance\"] = self.account_balance\n",
    "        long_short_dict[\"balance\"] = self.get_scaled_values(\"Account Balance\", self.current_trade[\"balance\"])\n",
    "        long_short_dict[\"profit\"] = self.get_scaled_values(\"Profit\", self.current_trade[\"profit\"])\n",
    "        long_short_dict[\"now\"] = self.current_step - 1\n",
    "        long_short_dict[\"next\"] = self.current_step\n",
    "        long_short_dict[\"holding\"] = self.get_scaled_values(\"Holding\", self.holding)\n",
    "        \n",
    "        return next_state, long_short_dict\n",
    "\n",
    "    def get_observation(self, time, action, account_balance, profit, holding, dummy_dataset = False):\n",
    "        #m5long\n",
    "        long_short_dict = {}\n",
    "        \n",
    "        m5long_last_image_path    =       self.forex_m5long.iloc[time-3][\"image_path\"]\n",
    "        m5long_third_image_path   =       self.forex_m5long.iloc[time-2][\"image_path\"]\n",
    "        m5long_second_image_path  =       self.forex_m5long.iloc[time-1][\"image_path\"]\n",
    "        m5long_current_image_path =       self.forex_m5long.iloc[time][\"image_path\"]\n",
    "        #save\n",
    "        #m5short\n",
    "        m5short_last_image_path    =       self.forex_m5short.iloc[time-3][\"image_path\"]\n",
    "        m5short_third_image_path   =       self.forex_m5short.iloc[time-2][\"image_path\"]\n",
    "        m5short_second_image_path  =       self.forex_m5short.iloc[time-1][\"image_path\"]\n",
    "        m5short_current_image_path =       self.forex_m5short.iloc[time][\"image_path\"]\n",
    "        #m1long\n",
    "        m1long_last_image_path    =       self.forex_m1long.iloc[time-3][\"image_path\"]        \n",
    "        m1long_third_image_path   =       self.forex_m1long.iloc[time-2][\"image_path\"]        \n",
    "        m1long_second_image_path  =       self.forex_m1long.iloc[time-1][\"image_path\"]        \n",
    "        m1long_current_image_path =       self.forex_m1long.iloc[time][\"image_path\"]\n",
    "        #save \n",
    "        m1short_last_image_path    =       self.forex_m1short.iloc[time-3][\"image_path\"]        \n",
    "        m1short_third_image_path   =       self.forex_m1short.iloc[time-2][\"image_path\"]\n",
    "        m1short_second_image_path  =       self.forex_m1short.iloc[time-1][\"image_path\"]\n",
    "        m1short_current_image_path =       self.forex_m1short.iloc[time][\"image_path\"]\n",
    "        #create image representation for this\n",
    "        if dummy_dataset == False:\n",
    "            m5long_last_image         =       self.read_image(m5long_last_image_path)\n",
    "            m5long_third_image        =       self.read_image(m5long_third_image_path)\n",
    "            m5long_second_image       =       self.read_image(m5long_second_image_path)\n",
    "            m5long_current_image      =       self.read_image(m5long_current_image_path)\n",
    "\n",
    "            m5short_last_image         =       self.read_image(m5short_last_image_path)\n",
    "            m5short_third_image        =       self.read_image(m5short_third_image_path)\n",
    "            m5short_second_image       =       self.read_image(m5short_second_image_path)\n",
    "            m5short_current_image      =       self.read_image(m5short_current_image_path)    \n",
    "            \n",
    "            m1long_last_image         =       self.read_image(m1long_last_image_path)\n",
    "            m1long_third_image        =       self.read_image(m1long_third_image_path)\n",
    "            m1long_second_image       =       self.read_image(m1long_second_image_path)\n",
    "            m1long_current_image      =       self.read_image(m1long_current_image_path)\n",
    "            \n",
    "            m1short_last_image         =       self.read_image(m1short_last_image_path)\n",
    "            m1short_third_image        =       self.read_image(m1short_third_image_path)\n",
    "            m1short_second_image       =       self.read_image(m1short_second_image_path)\n",
    "            m1short_current_image      =       self.read_image(m1short_current_image_path)          \n",
    "            array8 = np.broadcast_to(np.array([account_balance, profit, holding]).reshape(1,-1).reshape((1,1,-1)),(224,224,3))\n",
    "            stacked_image   =       np.stack([\n",
    "                                        m5long_last_image, m5long_third_image, m5long_second_image, m5long_current_image,\n",
    "                                        m5short_last_image, m5short_third_image, m5short_second_image, m5short_current_image,\n",
    "                                        m1long_last_image, m1long_third_image, m1long_second_image, m1long_current_image,\n",
    "                                        m1short_last_image, m1short_third_image, m1short_second_image, m1short_current_image, array8], axis=2)\n",
    "            stacked_image   =       stacked_image.transpose((2,0,1,3))\n",
    "            stacked_image   =       np.expand_dims(stacked_image, axis=0)\n",
    "            \n",
    "        state_features = [account_balance, profit, holding, action]\n",
    "        long_short_dict[\"state_features\"] = state_features\n",
    "        long_short_dict[\"holding\"] = holding\n",
    "        long_short_dict[\"balance\"] = account_balance\n",
    "        long_short_dict[\"account_balance\"] = self.account_balance\n",
    "        long_short_dict[\"now\"] = time\n",
    "        long_short_dict[\"next\"] = time + 1\n",
    "        long_short_dict[\"reward\"] = 0\n",
    "        long_short_dict[\"action\"] = action\n",
    "        long_short_dict[\"profit\"] = profit\n",
    "        long_short_dict[\"done\"] = self.done\n",
    "        \n",
    "        if dummy_dataset:\n",
    "            return [], long_short_dict\n",
    "        else:\n",
    "            return stacked_image, long_short_dict\n",
    "    def get_scaled_values(self, key, value):\n",
    "        if key == \"Account Balance\":return self.account_balance_norm.transform(np.array(float(value)).reshape(-1,1))[0][0]\n",
    "        elif key == \"Holding\":        return self.holding_norm.transform(np.array(float(value)).reshape(-1,1))[0][0]\n",
    "        elif key == \"Profit\":         return self.profit_norm.transform(np.array(float(value)).reshape(-1,1))[0][0]\n",
    "        elif key == \"Action\":         return self.action_norm.transform(np.array(float(value)).reshape(-1,1))[0][0]        \n",
    "    \n",
    "    def init_scalers(self):\n",
    "        self.account_balance_norm.fit(np.array([-1000,0,5000]).reshape(-1,1))\n",
    "        self.holding_norm.fit(np.array([0,2]).reshape(-1,1))\n",
    "        self.profit_norm.fit(np.array([-400,0,400]).reshape(-1,1))\n",
    "        self.action_norm.fit(np.array([0,2]).reshape(-1,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "UZjaBDMsGAeF"
   },
   "source": [
    "The agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "id": "jnOqzLKMFxUt",
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "class Agent:\n",
    "    def __init__(self, state_size, action_size, env):\n",
    "        self.model_name = \"v19_3D_3States\"\n",
    "        self.database = \"v19_3D_3States\"\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.gamma = 0.99\n",
    "        self.alpha = 0.00025\n",
    "        self.epsilon = 1.0\n",
    "        self.epsilon_min = 0.1\n",
    "        self.epsilon_decay = 0.995\n",
    "        self.update_rate = 1000\n",
    "        self.timesteps = 1\n",
    "        self.target_update_rate = 0.06\n",
    "        self.working_directory = \"colab/\"#\"/content/drive/MyDrive/Controller32/colab/\"\n",
    "        self.model_directory = self.working_directory + \"memories/\"\n",
    "        self.actor_model = self._build_actor_model()\n",
    "        self.critic_model = self._build_critic_model()\n",
    "        self.memory_indexer = 0\n",
    "        #Initialize the target models with sam\n",
    "        self.target_actor_model = self._build_actor_model()\n",
    "        self.target_critic_model = self._build_critic_model()\n",
    "        self.target_critic_model.set_weights(self.critic_model.get_weights())\n",
    "        self.env = env\n",
    "        self.remote = True\n",
    "        self.__setup__()\n",
    "        self.__epsilon_load__()\n",
    "    \n",
    "    def __epsilon_load__(self):\n",
    "        self.connect()\n",
    "        self.cursor.execute(f\"SELECT epsilon FROM history WHERE   ID = (SELECT MAX(id)  FROM history)\")\n",
    "        row = self.cursor.fetchone()\n",
    "        if row:\n",
    "            self.epsilon = row[0]\n",
    "        self.shutdown()\n",
    "    def __setup__(self, tables = [\"memory\", \"specialmemory\", \"history\"]):\n",
    "        for t in tables:\n",
    "            self.connect()\n",
    "            self.create_tables(t)\n",
    "            print(f\"Created {t} table\")\n",
    "        self.shutdown()\n",
    "        print(f\"Closing database\")\n",
    "        time.sleep(5)\n",
    "    def create_tables(self, table_name):\n",
    "        #to do add primary key to this table\n",
    "        if table_name == \"history\":\n",
    "            self.cursor.execute(f'''CREATE TABLE IF NOT EXISTS {table_name} (\n",
    "                id INTEGER PRIMARY KEY AUTOINCREMENT,\n",
    "                date DATE,\n",
    "                name TEXT,\n",
    "                memory_indexer INTEGER, \n",
    "                epsilon REAL\n",
    "                )''')\n",
    "        else:\n",
    "            self.cursor.execute(f'''CREATE TABLE IF NOT EXISTS {table_name} (\n",
    "                id INTEGER PRIMARY KEY AUTOINCREMENT,\n",
    "                long_short_dict TEXT,\n",
    "                long_short_dict_2 TEXT,\n",
    "                action REAL, \n",
    "                now INTEGER,\n",
    "                next INTEGER,            \n",
    "                count INTEGER,\n",
    "                priority INTEGER\n",
    "                )''')\n",
    "        self.connection.commit()\n",
    "\n",
    "    def connect(self):\n",
    "        #remote\n",
    "        if self.remote:\n",
    "            self.connection = sqlite3.connect(f\"{self.working_directory}{self.database}.db\")\n",
    "        #local\n",
    "        else:\n",
    "            self.connection = sqlite3.connect(f\"{self.working_directory}/{self.database}.db\")\n",
    "        self.cursor = self.connection.cursor()\n",
    "    def shutdown(self):\n",
    "        self.cursor.close()\n",
    "        self.connection.close()\n",
    "\n",
    "    def _build_base_model(self):\n",
    "        model = Sequential()\n",
    "        model.add(tf.keras.layers.Conv3D(32, (3,3,3), strides=(1,4,4), padding=\"same\", input_shape=self.state_size))\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(tf.keras.layers.MaxPooling3D(pool_size=(1,2,2), strides=(1,2,2), padding='same'))\n",
    "        model.add(tf.keras.layers.Conv3D(64, (3,3,3), padding=\"same\"))\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(tf.keras.layers.MaxPooling3D(pool_size=(1,2,2), strides=(1,2,2), padding='same'))\n",
    "        model.add(tf.keras.layers.Conv3D(64, (3,3,3), padding=\"same\"))\n",
    "        model.add(Activation('relu'))\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(128, activation='relu'))\n",
    "        model.add(Dense(128, activation='relu'))\n",
    "        model.add(Dense(64, activation='relu'))\n",
    "        model.add(Dense(32, activation='relu'))\n",
    "        return model\n",
    "    # Build the actor model responsible for predicting\n",
    "    \n",
    "    def _build_actor_model(self):\n",
    "        model = self._build_base_model()\n",
    "        model.add(Dense(self.action_size, activation='softmax'))\n",
    "        model.compile(loss='categorical_crossentropy', optimizer=RMSprop(learning_rate=self.alpha, rho=0.95, epsilon=None))\n",
    "        return model\n",
    "\n",
    "    \n",
    "    #Build the critic model responsible for criticising and calculating the risk of the reward\n",
    "    def _build_critic_model(self):\n",
    "        model = self._build_base_model()\n",
    "        model.add(Dense(self.action_size, activation='softmax'))\n",
    "        model.add(Dense(1, activation='linear'))\n",
    "        model.compile(loss='mse', optimizer=RMSprop(learning_rate=self.alpha, rho=0.95, epsilon=None))\n",
    "        \n",
    "        return model\n",
    "\n",
    "    def replay_v3(self, batch_size = 128, generate_dataset = False ):\n",
    "        update_weights = 0\n",
    "        break_while = True\n",
    "        while(break_while):\n",
    "            try:\n",
    "                if generate_dataset:\n",
    "                    self.dataset_generator(save=True,show_image=False, show_info = False ,_clear_output = True , start = 0, end = 5 , step = 1)    \n",
    "                self.connect()\n",
    "                print(\"Relay Connected\")\n",
    "                self.cursor.execute(f\"SELECT * FROM memory WHERE action = 0 ORDER BY RANDOM() LIMIT {batch_size*2}\")\n",
    "                rows = self.cursor.fetchall()\n",
    "                # print(\"Relay Connect ed to rows for action = 0 len rows\", len(rows))\n",
    "                self.cursor.execute(f\"SELECT * FROM memory WHERE action = 0.5 ORDER BY RANDOM() LIMIT {batch_size*2}\")\n",
    "                rows += self.cursor.fetchall()\n",
    "                # print(\"Relay Connect ed to rows for action = 2 len rows\", len(rows))\n",
    "                self.cursor.execute(f\"SELECT * FROM memory WHERE action = 1 ORDER BY RANDOM() LIMIT {batch_size*2}\")\n",
    "                rows += self.cursor.fetchall()                \n",
    "                print(\"Relay Connected to rows for action = 0 len rows\", len(rows))    \n",
    "                self.cursor.execute(f\"SELECT * FROM specialmemory WHERE action = 0 ORDER BY RANDOM() LIMIT {batch_size*2}\")\n",
    "                rows += self.cursor.fetchall()\n",
    "                print(\"Relay Connected to rows for action = 0 len rows\", len(rows))\n",
    "                self.cursor.execute(f\"SELECT * FROM specialmemory WHERE action = 0.5 ORDER BY RANDOM() LIMIT {batch_size*2}\")\n",
    "                rows += self.cursor.fetchall()\n",
    "                self.cursor.execute(f\"SELECT * FROM specialmemory WHERE action = 1 ORDER BY RANDOM() LIMIT {batch_size*2}\")\n",
    "                rows += self.cursor.fetchall()\n",
    "                print(\"Relay Connected to rows for action = 0 len rows\", len(rows))\n",
    "                self.shutdown()\n",
    "\n",
    "                random.shuffle(rows)\n",
    "                print(f\"length of rows{len(rows)}\")\n",
    "                for i,record in enumerate(rows):            \n",
    "                    # print(\"records\",record)\n",
    "                    long_short_dict_now = json.loads(record[1])\n",
    "                    long_short_dict_next = json.loads(record[2])\n",
    "                    state , _ = self.env.get_observation(record[4], long_short_dict_now[\"action\"], long_short_dict_now[\"balance\"], long_short_dict_now[\"profit\"], long_short_dict_now[\"holding\"])\n",
    "                    next_state, _ = self.env.get_observation(record[5], long_short_dict_next[\"action\"], long_short_dict_next[\"balance\"], long_short_dict_next[\"profit\"], long_short_dict_next[\"holding\"]) #np.expand_dims(current_stacked, axis=0)\n",
    "                    \n",
    "                    reward = long_short_dict_now[\"reward\"]\n",
    "                    try:\n",
    "                        done = long_short_dict_now[\"done\"]\n",
    "                    except:\n",
    "                        try:\n",
    "                            done = long_short_dict_now[\"info\"][\"done\"]\n",
    "                        except:\n",
    "                            print(\"Exception for Done not found\")\n",
    "                            exit(1)\n",
    "                    target = reward\n",
    "                    action = int(self.env.action_norm.inverse_transform([[record[3]]])[0][0])\n",
    "\n",
    "                    headers = [\"index\",\"Variations\", \"Probs\" ,\"rewards\", \"Actions\"]\n",
    "                    rows_display = []            \n",
    "                    if not done:\n",
    "                        target = (reward + self.gamma * self.target_critic_model.predict(next_state)[0][0])\n",
    "                    target_f = self.actor_model.predict(state)\n",
    "                    row = [i, \"target_f\", \",\".join(str(\"{:.2f}\".format(x)) for x in list(target_f[0])), target]\n",
    "                    rows_display.append(row)\n",
    "                    target_f[0][action] = target\n",
    "                    row = [i, \"suggested\",  \",\".join(str(\"{:.2f}\".format(x)) for x in list(target_f[0])), action]\n",
    "                    rows_display.append(row)\n",
    "                    \n",
    "                    #You need to test after every 64 epochs\n",
    "                    #test an episode with a certain number of lists\n",
    "                    if i % 128 == 0 and i > 0 :\n",
    "                        self.dataset_generator(save=False,show_image=False, show_info = True ,_clear_output = True , start = 50, end=100, step = 50)\n",
    "                        self.actor_model.save_weights(self.model_directory + self.model_name +\"_\"+ str(self.memory_indexer+1) + \"_actor.h5\")\n",
    "                        self.critic_model.save_weights(self.model_directory + self.model_name + \"_\" +str(self.memory_indexer)+ \"_critic.h5\")\n",
    "\n",
    "                    self.actor_model.fit(state, target_f, epochs=1, verbose=0)\n",
    "                    target_critic = np.array([[target]])\n",
    "                    self.critic_model.fit(state, target_critic, epochs=1, verbose=0)\n",
    "\n",
    "                    row = [i, \"final\", \",\".join(str(\"{:.2f}\".format(x)) for x in list(self.actor_model.predict(state)[0])), self.critic_model.predict(state, verbose=0)[0]]\n",
    "                    rows_display.append(row)\n",
    "\n",
    "                    print(tabulate(rows_display,headers=headers, tablefmt='grid'))\n",
    "\n",
    "                if update_weights % 800 == 0 and update_weights > 0:\n",
    "                    online_critic_weights = self.critic_model.get_weights()\n",
    "                    target_critic_weights = self.target_critic_model.get_weights()\n",
    "                    \n",
    "                    for i in range(len(target_critic_weights)):\n",
    "                        target_critic_weights[i] = self.target_update_rate * online_critic_weights[i] + (1 - self.target_update_rate) * target_critic_weights[i]\n",
    "                    self.target_critic_model.set_weights(target_critic_weights)\n",
    "                    self.connect()\n",
    "                    self.cursor.execute(\"INSERT INTO history VALUES (NULL, ?, ?, ?, ?)\",tuple([datetime.datetime.now(), self.model_name, self.memory_indexer, self.epsilon]))\n",
    "                    self.connection.commit()\n",
    "                    self.shutdown()\n",
    "                    self.save_model()\n",
    "                    print(\"\\n\\n\\nModel Weights Updated\\n\\n\\n\")\n",
    "\n",
    "                update_weights += 1\n",
    "                self.memory_indexer += 1\n",
    "                clear_output(wait=True)\n",
    "\n",
    "                if self.epsilon >= agent.epsilon_min:\n",
    "                    agent.epsilon *= agent.epsilon_decay\n",
    "\n",
    "            except KeyboardInterrupt:\n",
    "                self.shutdown()\n",
    "                self.save_model()\n",
    "                time.sleep(10)\n",
    "                break_while = False\n",
    "                rows=[]\n",
    "                state_values=[]\n",
    "                nextstate_values=[]\n",
    "                state = []\n",
    "                next_state=[]\n",
    "            #     clear_output(wait=True)\n",
    "            except Exception as e:\n",
    "                print(f\"\\nException in Replay : \\n {e} \\n\") \n",
    "                pass\n",
    "                self.shutdown()\n",
    "                self.save_model()\n",
    "                time.sleep(10)\n",
    "                break_while = False\n",
    "                rows=[]\n",
    "                state_values=[]\n",
    "                nextstate_values=[]\n",
    "                state = []\n",
    "                next_state=[]\n",
    "                clear_output(wait=True)\n",
    "    \n",
    "    def remember_v3(self, long_short_dict_now,long_short_dict_next,count = 0, priority=0, dummy_dataset = False):\n",
    "        row = []\n",
    "        row.append(json.dumps(long_short_dict_now))\n",
    "        row.append(json.dumps(long_short_dict_next))\n",
    "        row.append(long_short_dict_now[\"action\"])\n",
    "        row.append(long_short_dict_now[\"now\"])\n",
    "        row.append(long_short_dict_now[\"next\"])\n",
    "        row.append(count)\n",
    "        row.append(priority)\n",
    "        if dummy_dataset:\n",
    "            if long_short_dict_now[\"reward\"] > 0.02 :\n",
    "                self.cursor.execute(\"INSERT INTO specialmemory VALUES (NULL, ?, ?, ?, ?, ?, ?, ? )\",tuple(row))\n",
    "                self.connection.commit()\n",
    "            # else:\n",
    "            #     self.cursor.execute(\"INSERT INTO memory VALUES (NULL, ?, ?, ?, ?, ?, ?, ? )\",tuple(row))\n",
    "            #     self.connection.commit()            \n",
    "        else:\n",
    "            self.connect()\n",
    "            if long_short_dict_now[\"reward\"] > 0.02 :\n",
    "                self.cursor.execute(\"INSERT INTO specialmemory VALUES (NULL, ?, ?, ?, ?, ?, ?, ? )\",tuple(row))\n",
    "                self.connection.commit()\n",
    "            # else:\n",
    "            #     self.cursor.execute(\"INSERT INTO memory VALUES (NULL, ?, ?, ?, ?, ?, ?, ? )\",tuple(row))\n",
    "            #     self.connection.commit()\n",
    "            self.shutdown()\n",
    "        \n",
    "    def act(self, state):\n",
    "        if np.random.rand() <= self.epsilon:\n",
    "            return secrets.randbelow(self.action_size) if np.random.choice(20) % 2 == 0 else np.random.choice(self.action_size) , [[0,0,0]]\n",
    "        else:\n",
    "            pred = self.actor_model.predict(state)\n",
    "            return np.argmax(pred[0]), pred   \n",
    "   \n",
    "    def predict(self, state):\n",
    "        return self.actor_model.predict(state)\n",
    "\n",
    "    # Load the models\n",
    "    def load(self):\n",
    "        self.actor_model.load_weights(self.model_directory + self.model_name + str(self.memory_indexer) + \"_actor.h5\")\n",
    "        self.critic_model.load_weights(self.model_directory + self.model_name + str(self.memory_indexer) + \"_critic.h5\")\n",
    "        self.target_critic_model.set_weights(self.critic_model.get_weights())\n",
    "    \n",
    "    # Save the models\n",
    "    def save_model(self):\n",
    "        self.actor_model.save_weights(self.model_directory + self.model_name + str(self.memory_indexer) + \"_actor.h5\")\n",
    "        self.critic_model.save_weights(self.model_directory + self.model_name + str(self.memory_indexer) + \"_critic.h5\")\n",
    "\n",
    "        \n",
    "    def read_image(self, image_path):\n",
    "        return np.asarray(Image.open(image_path))/255\n",
    "    '''\n",
    "        this function generates random environment noise for the ai\n",
    "        it takes save parameter if you want the random data to be saved or not. \n",
    "        also this function can run the environment to see how the ai behaves. It can print out how the learning is progressing\n",
    "    '''\n",
    "    def dataset_generator(self, save = False, show_image=False, show_info=True, _clear_output=False, start=0, end=20, step = 2):\n",
    "        print(\"Database generator\")\n",
    "        episode = 0\n",
    "        for epoch in list(range(start, end, step)):\n",
    "            state, long_short_dict_now = self.env.reset(dummy_dataset = True)\n",
    "            headers = [\"index\",\"Action\",\"epsilon\",\"Pos\",\"done\",\"reward\",\"E Price\",\"C Price\", \"takep\",\"stopLoss\",\"profit\" ,\"balance\",\"pred\"]\n",
    "            self.env.current_step = 4\n",
    "            self.env.current_end = 1226\n",
    "            print(f\"episode {episode}\")\n",
    "            for index in list(range(4,1226, 1)):\n",
    "                rows = [] \n",
    "                row = []    \n",
    "                action, pred = secrets.randbelow(self.action_size) if np.random.choice(20) % 2 == 0 else np.random.choice(self.action_size), [[0,0,0]]#self.act(state)\n",
    "                next_state, long_short_dict_next = self.env.step(action, dummy_dataset = True)\n",
    "                reward = long_short_dict_next[\"reward\"]\n",
    "                if save:\n",
    "                    self.remember_v3(long_short_dict_now , long_short_dict_next, 0,0)                    \n",
    "                state = next_state\n",
    "                if show_image:\n",
    "                    self.display_row_images(long_short_dict_now, long_short_dict_next)\n",
    "                long_short_dict_now = long_short_dict_next\n",
    "                row = [index,action,self.epsilon,long_short_dict_now[\"holding\"], long_short_dict_now[\"done\"], \"{:.2f}\".format(reward), long_short_dict_now[\"info\"][\"c_trade\"][\"entry_price\"],long_short_dict_now[\"info\"][\"c_trade\"][\"current_price\"],long_short_dict_now[\"info\"][\"c_trade\"][\"take_profit\"],long_short_dict_now[\"info\"][\"c_trade\"][\"stop_loss\"], long_short_dict_now[\"info\"][\"c_trade\"][\"profit\"],long_short_dict_now[\"balance\"], \",\".join(str(\"{:.2f}\".format(x)) for x in list(pred[0]))]\n",
    "                row_totals = [\"_step\",long_short_dict_now[\"info\"][\"_step\"]-1,\"\",\"\",\"\",\"Total\",5 * long_short_dict_now[\"info\"][\"c_trade\"][\"profit\"],\"\",\"\"]\n",
    "                rows.append(row)\n",
    "                rows.append(row_totals)\n",
    "                if show_image:\n",
    "                    self.display_row_images(long_short_dict_now, long_short_dict_next)\n",
    "                if show_info:\n",
    "                    print(tabulate(rows, headers=headers, tablefmt='grid'))\n",
    "                if long_short_dict_now[\"done\"] == 1 or index > 1225:\n",
    "                    done=0\n",
    "                    print(\"Episode {} \".format(episode), end=\"\\r\")\n",
    "                    state, long_short_dict_now = self.env.reset()\n",
    "                    clear_output(wait=True)\n",
    "                    break\n",
    "            episode += 1\n",
    "            # print(f\"Episode {episode}\")\n",
    "            if _clear_output:\n",
    "                clear_output(wait=True)\n",
    "    def display_row_images(self,long_short_dict_now , long_short_dict_next):\n",
    "        imageq = Image.open(f\"{self.env.dataset_directory}{self.env.forex_m1short.iloc[long_short_dict_now['now']-3]['image_path']}\")\n",
    "        imaget = Image.open(f\"{self.env.dataset_directory}{self.env.forex_m1long.iloc[long_short_dict_now['now']-3]['image_path']}\")\n",
    "        images = Image.open(f\"{self.env.dataset_directory}{self.env.forex_m5long.iloc[long_short_dict_now['now']-3]['image_path']}\")\n",
    "        imagec = Image.open(f\"{self.env.dataset_directory}{self.env.forex_m5short.iloc[long_short_dict_now['now']]['image_path']}\")\n",
    "        # Calculate the required dimensions for the new image\n",
    "        new_width = imageq.width * 4\n",
    "        new_height = imageq.height\n",
    "        # Create a new image with the required dimensions\n",
    "        new_image = Image.new(\"RGB\", (new_width, new_height))\n",
    "        # Paste the individual images side by side\n",
    "        new_image.paste(imageq, (0, 0))\n",
    "        new_image.paste(imaget, (imageq.width, 0))\n",
    "        new_image.paste(images, (imageq.width + imaget.width, 0))\n",
    "        new_image.paste(imagec, (imageq.width + imaget.width + images.width, 0))\n",
    "        # Display the final imagem5short_image_path\n",
    "        \n",
    "        display(new_image) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# env = ForexEnv(m5long, m5short, m1long, m1short)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "id": "3lRP-UA5NJPj",
    "tags": []
   },
   "outputs": [],
   "source": [
    "#open datasets\n",
    "# m5long = pd.read_csv(\"/content/drive/MyDrive/Controller32/final_m5long.csv\")\n",
    "# m5short = pd.read_csv(\"/content/drive/MyDrive/Controller32/final_m5short.csv\")\n",
    "# m1long = pd.read_csv(\"/content/drive/MyDrive/Controller32/final_m1long.csv\")\n",
    "# m1short = pd.read_csv(\"/content/drive/MyDrive/Controller32/final_m1short.csv\")\n",
    "m5long = pd.read_csv(\"final_m5long.csv\")\n",
    "m5short = pd.read_csv(\"final_m5short.csv\")\n",
    "m1long = pd.read_csv(\"final_m1long.csv\")\n",
    "m1short = pd.read_csv(\"final_m1short.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "DCDBuYbs6kVZ",
    "outputId": "8a34f29b-63ee-4393-ecb7-ed378d24b06a",
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created memory table\n",
      "Created specialmemory table\n",
      "Created history table\n",
      "Closing database\n"
     ]
    }
   ],
   "source": [
    "env = ForexEnv(m5long, m5short, m1long, m1short)\n",
    "state_size=(17,224,224, 3)\n",
    "action_size = 3\n",
    "agent = Agent(state_size, action_size, env)\n",
    "agent.alpha = 0.00025\n",
    "agent.memory_indexer = 0\n",
    "agent.epsilon = 1.0\n",
    "# print(agent.critic_model.summary())\n",
    "# agent.actor_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "sZYFuLHkN8tM",
    "tags": []
   },
   "outputs": [],
   "source": [
    "agent.dataset_generator(save=True, show_image = False, _clear_output=True, show_info=True, start= 1, end=20, step=1)\n",
    "\n",
    "# agent.replay_v3()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "agent.replay_v3(32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def dataset_generator(agent,save = False, show_image=False, show_info=True, _clear_output=False, start=0, end=20, step = 2):\n",
    "    print(\"Database generator\")\n",
    "    agent.epsilon = 0.0001\n",
    "    episode = 0\n",
    "    memo = 0\n",
    "    for epoch in list(range(start, end, step)):\n",
    "        state, long_short_dict_now = agent.env.reset(dummy_dataset = False)\n",
    "        headers = [\"index\",\"Action\",\"epsilon\",\"Pos\",\"done\",\"reward\",\"E Price\",\"C Price\", \"takep\",\"stopLoss\",\"profit\" ,\"balance\",\"pred\"]\n",
    "        agent.env.current_step = 4\n",
    "        agent.env.current_end = 1226\n",
    "        print(f\"episode {episode}\")\n",
    "        for index in list(range(4,1226, 1)):\n",
    "            rows = [] \n",
    "            row = []\n",
    "            print(state.shape)\n",
    "            print(agent.act(state))\n",
    "            action, pred = agent.act(state)#secrets.randbelow(self.action_size) if np.random.choice(20) % 2 == 0 else np.random.choice(self.action_size), [[0,0,0]]#self.act(state)\n",
    "            \n",
    "            next_state, long_short_dict_next = agent.env.step(action, dummy_dataset = False)\n",
    "            reward = long_short_dict_next[\"reward\"]\n",
    "            if save:\n",
    "                agent.remember_v3(long_short_dict_now , long_short_dict_next, 0,0)                    \n",
    "            state = next_state\n",
    "            if show_image:\n",
    "                agent.display_row_images(long_short_dict_now, long_short_dict_next)\n",
    "            long_short_dict_now = long_short_dict_next\n",
    "            row = [index,action,agent.epsilon,long_short_dict_now[\"holding\"], long_short_dict_now[\"done\"], \"{:.2f}\".format(reward), long_short_dict_now[\"info\"][\"c_trade\"][\"entry_price\"],long_short_dict_now[\"info\"][\"c_trade\"][\"current_price\"],long_short_dict_now[\"info\"][\"c_trade\"][\"take_profit\"],long_short_dict_now[\"info\"][\"c_trade\"][\"stop_loss\"], long_short_dict_now[\"info\"][\"c_trade\"][\"profit\"],long_short_dict_now[\"balance\"], \",\".join(str(\"{:.2f}\".format(x)) for x in list(pred[0]))]\n",
    "            row_totals = [\"_step\",long_short_dict_now[\"info\"][\"_step\"]-1,\"\",\"\",\"\",\"Total\",5 * long_short_dict_now[\"info\"][\"c_trade\"][\"profit\"],\"\",\"\"]\n",
    "            rows.append(row)\n",
    "            rows.append(row_totals)\n",
    "            if show_image:\n",
    "                agent.display_row_images(long_short_dict_now, long_short_dict_next)\n",
    "            if show_info:\n",
    "                print(tabulate(rows, headers=headers, tablefmt='grid'))\n",
    "            if long_short_dict_now[\"done\"] == 1 or index > 1225:\n",
    "                done=0\n",
    "                print(\"Episode {} \".format(episode), end=\"\\r\")\n",
    "                state, long_short_dict_now = agent.env.reset()\n",
    "                clear_output(wait=True)\n",
    "                break\n",
    "                \n",
    "        episode += 1\n",
    "        # print(f\"Episode {episode}\")\n",
    "        if _clear_output:\n",
    "            clear_output(wait=True)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dir(agent.save_model())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataset_generator(agent, start=1 , end=3, step=1, show_info=True, show_image=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created memory table\n",
      "Created specialmemory table\n",
      "Created history table\n",
      "Closing database\n"
     ]
    }
   ],
   "source": [
    "agent.__setup__()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dummy dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m5long = pd.read_csv(\"final_m5long.csv\")\n",
    "m5short = pd.read_csv(\"final_m5short.csv\")\n",
    "m1long = pd.read_csv(\"final_m1long.csv\")\n",
    "m1short = pd.read_csv(\"final_m1short.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Positive dataset generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "# time.sleep(10)\n",
    "agent.epsilon = 0.003013798250576# 1.0#0.0003# 0.46211964903917074#1.0#0.00996820918179746\n",
    "batch_size = 32\n",
    "done = False\n",
    "info = {}\n",
    "reward = 0\n",
    "episode = 0\n",
    "state, long_short_dict = env.reset(dummy_dataset=True)\n",
    "\n",
    "headers = [\"index\",\"Action\",\"epsilon\",\"Pos\",\"done\",\"reward\",\"E Price\",\"C Price\", \"takep\",\"highest\",\"profit\" ,\"balance\",\"pred\"]\n",
    "break_while = True\n",
    "show_image = False\n",
    "show_info = True\n",
    "counter = -1\n",
    "\n",
    "def generate_sequence(length, check=False, key=0):\n",
    "    start_value = str(random.randint(0, 2))\n",
    "    if check:\n",
    "        start_value = key\n",
    "    if start_value == 0:\n",
    "        sequence = [0] * (length) \n",
    "    elif start_value == 1:\n",
    "        sequence = [1] * (length - 1) + [0]\n",
    "    elif start_value == 2:\n",
    "        sequence = [2] * (length - 1) + [0]\n",
    "    return sequence\n",
    "\n",
    "steps = [10,20,30,40,50, 60, 70, 80, 90, 100, 110,120,130,140,150,160,170,180,190,200]\n",
    "actions = [1,2]\n",
    "total = list(range(4, 1226, 1))\n",
    "sections = [total[i:i+200] for i in range(0, len(total), 200)]\n",
    "delta = 0\n",
    "break_all = False\n",
    "stop_asking = True\n",
    "agent.connect()\n",
    "operations = 0\n",
    "for _action in actions:\n",
    "    if break_all:\n",
    "        break\n",
    "    for step in steps:\n",
    "        if break_all:\n",
    "            break\n",
    "        sequences = generate_sequence(step, True, _action) \n",
    "        for section in sections:\n",
    "            if break_all:\n",
    "                break\n",
    "            # print(f\"length: {len(section)} : {section}\\nSteps : {step}\\nsequence:{sequences}\\n\")\n",
    "            section_parts = []\n",
    "            if len(sequences) == 10:\n",
    "                section_parts = [section[i:i+10] for i in range(0, len(section), 10)]\n",
    "            elif len(sequences) == 20:\n",
    "                section_parts = [section[i:i+20] for i in range(0, len(section), 20)]\n",
    "            elif len(sequences) == 30:\n",
    "                section_parts = [section[i:i+30] for i in range(0, len(section), 30)]\n",
    "            elif len(sequences) == 40:\n",
    "                section_parts = [section[i:i+40] for i in range(0, len(section), 40)]\n",
    "            elif len(sequences) == 50:\n",
    "                section_parts = [section[i:i+50] for i in range(0, len(section), 50)]\n",
    "            elif len(sequences) == 60:\n",
    "                section_parts = [section[i:i+60] for i in range(0, len(section), 60)]                \n",
    "            elif len(sequences) == 70:\n",
    "                section_parts = [section[i:i+70] for i in range(0, len(section), 70)]\n",
    "            elif len(sequences) == 80:\n",
    "                section_parts = [section[i:i+80] for i in range(0, len(section), 80)]\n",
    "            elif len(sequences) == 90:\n",
    "                section_parts = [section[i:i+90] for i in range(0, len(section), 90)]\n",
    "            elif len(sequences) == 100:\n",
    "                section_parts = [section[i:i+100] for i in range(0, len(section), 100)]                \n",
    "            elif len(sequences) == 110:\n",
    "                section_parts = [section[i:i+110] for i in range(0, len(section), 110)]\n",
    "            elif len(sequences) == 120:\n",
    "                section_parts = [section[i:i+120] for i in range(0, len(section), 120)]\n",
    "            elif len(sequences) == 130:\n",
    "                section_parts = [section[i:i+130] for i in range(0, len(section), 130)]\n",
    "            elif len(sequences) == 140:\n",
    "                section_parts = [section[i:i+140] for i in range(0, len(section), 140)]\n",
    "            elif len(sequences) == 150:\n",
    "                section_parts = [section[i:i+150] for i in range(0, len(section), 150)]\n",
    "            elif len(sequences) == 160:\n",
    "                section_parts = [section[i:i+160] for i in range(0, len(section), 160)]\n",
    "            elif len(sequences) == 170:\n",
    "                section_parts = [section[i:i+150] for i in range(0, len(section), 170)]\n",
    "            elif len(sequences) == 180:\n",
    "                section_parts = [section[i:i+150] for i in range(0, len(section), 180)]\n",
    "            elif len(sequences) == 190:\n",
    "                section_parts = [section[i:i+150] for i in range(0, len(section), 190)]                \n",
    "            elif len(sequences) == 200:\n",
    "                section_parts = [section[i:i+200] for i in range(0, len(section), 200)] \n",
    "            print(sequences)\n",
    "            \n",
    "            # print(f\"length: {len(section)} : {section}\\nSteps : {step}\\nsequence:{sequences}\\n\")\n",
    "            # print(f\"section parts {len(section_parts)}\\n{section_parts}\")\n",
    "            # if len(section_parts) > 0 and step == 50:\n",
    "            #     # print(f\"\\nSection parts found {section_parts} \\nsection sequences {sequences}\\n\")\n",
    "            for parts in section_parts:\n",
    "                \n",
    "                env = ForexEnv(m5long, m5short, m1long, m1short)\n",
    "                state, long_short_dict_now = env.reset(dummy_dataset=True)\n",
    "                agent.env = env\n",
    "                agent.env.current_step = parts[0]\n",
    "                agent.env.current_end = parts[-1]\n",
    "                rows = []\n",
    "                for index, part in enumerate(parts):\n",
    "                    # print(index, part, sequences[index], sequences)\n",
    "                    action = sequences[index]\n",
    "                    try:\n",
    "                        next_state, long_short_dict_next = env.step(action,dummy_dataset=True)\n",
    "                        agent.remember_v3(long_short_dict_now , long_short_dict_next, 0,0, dummy_dataset=True)\n",
    "                        pred = [[0.35835683, 0.30909953, 0.33254367]]\n",
    "                        state = next_state\n",
    "                        long_short_dict_now = long_short_dict_next\n",
    "                        print(f\"Operations: {operations}, {agent.env.current_step}\", end=\"\\r\")\n",
    "                    except Exception as e:\n",
    "                        print(f\"Exception caught:\\n{e}\\n\", end = \"\\r\")\n",
    "                    operations += 1\n",
    "                    # row = []\n",
    "                    # row = [index,action,agent.epsilon,long_short_dict_now[\"holding\"], long_short_dict_now[\"done\"], \"{:.2f}\".format(reward), long_short_dict_now[\"info\"][\"c_trade\"][\"entry_price\"],long_short_dict_now[\"info\"][\"c_trade\"][\"current_price\"],long_short_dict_now[\"info\"][\"c_trade\"][\"take_profit\"],long_short_dict_now[\"info\"][\"c_trade\"][\"highest\"], long_short_dict_now[\"info\"][\"c_trade\"][\"profit\"],long_short_dict_now[\"balance\"], \",\".join(str(\"{:.2f}\".format(x)) for x in list(pred[0]))]\n",
    "                    # row_totals = [\"_step\",long_short_dict_now[\"info\"][\"_step\"]-2,\"\",\"\",\"\",\"Total\",5 * long_short_dict_now[\"info\"][\"c_trade\"][\"profit\"],\"\",\"\"]\n",
    "                    # rows.append(row)\n",
    "                    # rows.append(row_totals)\n",
    "                    # print(tabulate(rows, headers=headers, tablefmt='grid'))   \n",
    "                \n",
    "                # if stop_asking:\n",
    "                #     message = input(\"Do you want to continue\")\n",
    "                #     if message == \"no\":\n",
    "                #         break_all = True\n",
    "                #     elif message == \"yes\":\n",
    "                #         clear_output(wait=True)\n",
    "                #     elif message == \"stop_asking\":\n",
    "                #         stop_asking = False\n",
    "\n",
    "              \n",
    "                \n",
    "\n",
    "#             # clear_output(wait=True) \n",
    "agent.shutdown()\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "agent.shutdown()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = [304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397, 398, 399, 400, 401, 402, 403]\n",
    "len(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "500"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "add columns to dataset highest, duration and ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Relay Connected\n",
      "Relay Connected to rows for action = 0 len rows 0\n",
      "Relay Connected to rows for action = 0 len rows 0\n",
      "Relay Connected to rows for action = 0 len rows 128\n",
      "length of rows128\n",
      "1/1 [==============================] - 0s 137ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 156ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|       0 | target_f     | 0.00,0.00,1.00 |  0.582082 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       0 | suggested    | 0.00,0.58,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       0 | final        | 0.00,1.00,0.00 |  0.599077 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 142ms/step\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|       1 | target_f     | 0.00,1.00,0.00 |  0.736827 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       1 | suggested    | 0.00,0.74,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       1 | final        | 0.00,1.00,0.00 |  0.756413 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|       2 | target_f     | 0.00,1.00,0.00 |  0.621517 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       2 | suggested    | 0.00,0.62,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       2 | final        | 0.00,0.00,1.00 |  0.623092 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 152ms/step\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 152ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|       3 | target_f     | 0.00,0.00,1.00 |  0.591812 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       3 | suggested    | 0.00,0.00,0.59 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       3 | final        | 0.00,1.00,0.00 |  0.59824  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 143ms/step\n",
      "1/1 [==============================] - 0s 137ms/step\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|       4 | target_f     | 0.00,1.00,0.00 |  0.579379 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       4 | suggested    | 0.00,1.00,0.58 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       4 | final        | 0.00,0.00,1.00 |  0.599427 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 164ms/step\n",
      "1/1 [==============================] - 0s 158ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|       5 | target_f     | 0.00,0.00,1.00 |  0.622611 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       5 | suggested    | 0.00,0.00,0.62 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       5 | final        | 0.00,0.00,1.00 |  0.604807 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 138ms/step\n",
      "1/1 [==============================] - 0s 137ms/step\n",
      "1/1 [==============================] - 0s 206ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|       6 | target_f     | 0.00,0.00,1.00 |  0.6331   |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       6 | suggested    | 0.00,0.00,0.63 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       6 | final        | 0.00,1.00,0.00 |  0.620807 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 137ms/step\n",
      "1/1 [==============================] - 0s 138ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|       7 | target_f     | 0.00,1.00,0.00 |  0.591192 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       7 | suggested    | 0.00,0.59,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       7 | final        | 0.00,0.00,1.00 |  0.591413 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 142ms/step\n",
      "1/1 [==============================] - 0s 140ms/step\n",
      "1/1 [==============================] - 0s 153ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|       8 | target_f     | 0.00,0.00,1.00 |  0.641424 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       8 | suggested    | 0.00,0.00,0.64 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       8 | final        | 0.00,1.00,0.00 |  0.614639 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 156ms/step\n",
      "1/1 [==============================] - 0s 138ms/step\n",
      "1/1 [==============================] - 0s 155ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|       9 | target_f     | 0.00,1.00,0.00 |  0.676923 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       9 | suggested    | 0.00,1.00,0.68 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|       9 | final        | 0.00,0.00,1.00 |  0.676428 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 135ms/step\n",
      "1/1 [==============================] - 0s 153ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      10 | target_f     | 0.00,0.00,1.00 |  0.67744  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      10 | suggested    | 0.00,0.00,0.68 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      10 | final        | 0.00,0.00,1.00 |  0.667657 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      11 | target_f     | 0.00,0.00,1.00 |  0.728852 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      11 | suggested    | 0.00,0.73,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      11 | final        | 0.00,1.00,0.00 |  0.722284 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 165ms/step\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      12 | target_f     | 0.00,1.00,0.00 |  0.634808 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      12 | suggested    | 0.00,1.00,0.63 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      12 | final        | 0.00,1.00,0.00 |  0.602955 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 142ms/step\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "1/1 [==============================] - 0s 152ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      13 | target_f     | 0.00,1.00,0.00 |  0.584109 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      13 | suggested    | 0.00,1.00,0.58 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      13 | final        | 0.00,0.00,1.00 |  0.586896 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 155ms/step\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      14 | target_f     | 0.00,0.00,1.00 |  0.599333 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      14 | suggested    | 0.00,0.00,0.60 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      14 | final        | 0.00,0.00,1.00 |  0.609407 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 140ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "1/1 [==============================] - 0s 167ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      15 | target_f     | 0.00,0.00,1.00 |  0.676431 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      15 | suggested    | 0.00,0.68,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      15 | final        | 0.00,1.00,0.00 |  0.693698 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 152ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "1/1 [==============================] - 0s 176ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      16 | target_f     | 0.00,1.00,0.00 |  0.598023 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      16 | suggested    | 0.00,0.60,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      16 | final        | 0.00,1.00,0.00 |  0.601625 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 155ms/step\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "1/1 [==============================] - 0s 153ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      17 | target_f     | 0.00,1.00,0.00 |  0.729013 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      17 | suggested    | 0.00,0.73,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      17 | final        | 0.00,0.00,1.00 |  0.737113 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 142ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      18 | target_f     | 0.00,0.00,1.00 |  0.59565  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      18 | suggested    | 0.00,0.60,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      18 | final        | 0.00,1.00,0.00 |  0.650937 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 143ms/step\n",
      "1/1 [==============================] - 0s 163ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      19 | target_f     | 0.00,1.00,0.00 |  0.607294 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      19 | suggested    | 0.00,0.61,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      19 | final        | 0.00,1.00,0.00 |  0.605149 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 155ms/step\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      20 | target_f     | 0.00,1.00,0.00 |  0.607518 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      20 | suggested    | 0.00,1.00,0.61 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      20 | final        | 0.00,0.00,1.00 |  0.592583 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 137ms/step\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      21 | target_f     | 0.00,0.00,1.00 |  0.581528 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      21 | suggested    | 0.00,0.58,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      21 | final        | 0.00,0.00,1.00 |  0.626125 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      22 | target_f     | 0.00,0.00,1.00 |  0.57888  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      22 | suggested    | 0.00,0.58,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      22 | final        | 0.00,1.00,0.00 |  0.580095 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "1/1 [==============================] - 0s 152ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      23 | target_f     | 0.00,1.00,0.00 |  0.573476 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      23 | suggested    | 0.00,0.57,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      23 | final        | 0.00,1.00,0.00 |  0.576506 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "1/1 [==============================] - 0s 129ms/step\n",
      "1/1 [==============================] - 0s 144ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      24 | target_f     | 0.00,1.00,0.00 |  0.678724 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      24 | suggested    | 0.00,0.68,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      24 | final        | 0.00,0.00,1.00 |  0.635436 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 141ms/step\n",
      "1/1 [==============================] - 0s 143ms/step\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      25 | target_f     | 0.00,0.00,1.00 |  0.634472 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      25 | suggested    | 0.00,0.63,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      25 | final        | 0.00,1.00,0.00 |  0.628203 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 144ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      26 | target_f     | 0.00,1.00,0.00 |  0.602832 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      26 | suggested    | 0.00,0.60,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      26 | final        | 0.00,1.00,0.00 |  0.59249  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 136ms/step\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      27 | target_f     | 0.00,1.00,0.00 |  0.599333 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      27 | suggested    | 0.00,1.00,0.60 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      27 | final        | 0.00,0.00,1.00 |  0.605869 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "1/1 [==============================] - 0s 132ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      28 | target_f     | 0.00,0.00,1.00 |  0.596757 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      28 | suggested    | 0.00,0.60,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      28 | final        | 0.00,0.00,1.00 |  0.602847 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "1/1 [==============================] - 0s 132ms/step\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      29 | target_f     | 0.00,0.00,1.00 |  0.60367  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      29 | suggested    | 0.00,0.60,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      29 | final        | 0.00,1.00,0.00 |  0.592131 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 157ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      30 | target_f     | 0.00,1.00,0.00 |  0.03565  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      30 | suggested    | 0.00,1.00,0.04 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      30 | final        | 0.00,1.00,0.00 |  0.501714 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      31 | target_f     | 0.00,1.00,0.00 |  0.617389 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      31 | suggested    | 0.00,0.62,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      31 | final        | 0.00,1.00,0.00 |  0.596978 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 138ms/step\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      32 | target_f     | 0.00,1.00,0.00 |  0.052175 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      32 | suggested    | 0.00,1.00,0.05 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      32 | final        | 0.00,0.00,1.00 |  0.519194 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 138ms/step\n",
      "1/1 [==============================] - 0s 160ms/step\n",
      "1/1 [==============================] - 0s 166ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      33 | target_f     | 0.00,0.00,1.00 |  0.681305 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      33 | suggested    | 0.00,0.00,0.68 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      33 | final        | 0.00,1.00,0.00 |  0.557748 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 153ms/step\n",
      "1/1 [==============================] - 0s 137ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      34 | target_f     | 0.00,1.00,0.00 |  0.0322   |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      34 | suggested    | 0.00,1.00,0.03 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      34 | final        | 0.00,1.00,0.00 |  0.483104 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "1/1 [==============================] - 0s 139ms/step\n",
      "1/1 [==============================] - 0s 190ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      35 | target_f     | 0.00,1.00,0.00 |  0.606195 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      35 | suggested    | 0.00,1.00,0.61 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      35 | final        | 0.00,0.00,1.00 |  0.523599 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "1/1 [==============================] - 0s 141ms/step\n",
      "1/1 [==============================] - 0s 224ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      36 | target_f     | 0.00,0.00,1.00 |  0.611066 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      36 | suggested    | 0.00,0.00,0.61 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      36 | final        | 0.00,1.00,0.00 |  0.538486 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 142ms/step\n",
      "1/1 [==============================] - 0s 153ms/step\n",
      "1/1 [==============================] - 0s 136ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      37 | target_f     | 0.00,1.00,0.00 |  0.585466 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      37 | suggested    | 0.00,1.00,0.59 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      37 | final        | 0.00,0.00,1.00 |  0.545418 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 160ms/step\n",
      "1/1 [==============================] - 0s 152ms/step\n",
      "1/1 [==============================] - 0s 166ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      38 | target_f     | 0.00,0.00,1.00 |  0.57753  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      38 | suggested    | 0.00,0.00,0.58 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      38 | final        | 0.00,0.00,1.00 |  0.522676 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 158ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      39 | target_f     | 0.00,0.00,1.00 |  0.620086 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      39 | suggested    | 0.00,0.00,0.62 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      39 | final        | 0.00,1.00,0.00 |  0.52227  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 138ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      40 | target_f     | 0.00,1.00,0.00 |  0.64919  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      40 | suggested    | 0.00,0.65,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      40 | final        | 0.00,0.00,1.00 |  0.616502 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 144ms/step\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "1/1 [==============================] - 0s 155ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      41 | target_f     | 0.00,0.00,1.00 |  0.591579 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      41 | suggested    | 0.00,0.00,0.59 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      41 | final        | 0.00,1.00,0.00 |  0.530315 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 143ms/step\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      42 | target_f     | 0.00,1.00,0.00 |  0.583155 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      42 | suggested    | 0.00,1.00,0.58 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      42 | final        | 0.00,0.00,1.00 |  0.555483 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "1/1 [==============================] - 0s 159ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      43 | target_f     | 0.00,0.00,1.00 |  0.620878 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      43 | suggested    | 0.00,0.00,0.62 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      43 | final        | 0.00,1.00,0.00 |  0.571535 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 152ms/step\n",
      "1/1 [==============================] - 0s 137ms/step\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      44 | target_f     | 0.00,1.00,0.00 |  0.643496 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      44 | suggested    | 0.00,1.00,0.64 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      44 | final        | 0.00,0.00,1.00 |  0.519648 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 134ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 144ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      45 | target_f     | 0.00,0.00,1.00 |  0.601212 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      45 | suggested    | 0.00,0.60,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      45 | final        | 0.00,1.00,0.00 |  0.606476 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "1/1 [==============================] - 0s 153ms/step\n",
      "1/1 [==============================] - 0s 153ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      46 | target_f     | 0.00,1.00,0.00 |  0.644382 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      46 | suggested    | 0.00,1.00,0.64 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      46 | final        | 0.00,0.00,1.00 |  0.601002 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 152ms/step\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "1/1 [==============================] - 0s 163ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      47 | target_f     | 0.00,0.00,1.00 |  0.604364 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      47 | suggested    | 0.00,0.00,0.60 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      47 | final        | 0.00,0.00,1.00 |  0.585311 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 131ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 174ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      48 | target_f     | 0.00,0.00,1.00 |  0.567264 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      48 | suggested    | 0.00,0.57,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      48 | final        | 0.00,1.00,0.00 |  0.579974 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 142ms/step\n",
      "1/1 [==============================] - 0s 138ms/step\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      49 | target_f     | 0.00,1.00,0.00 |  0.57463  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      49 | suggested    | 0.00,1.00,0.57 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      49 | final        | 0.00,0.00,1.00 |  0.555553 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 152ms/step\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "1/1 [==============================] - 0s 139ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      50 | target_f     | 0.00,0.00,1.00 |  0.598884 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      50 | suggested    | 0.00,0.00,0.60 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      50 | final        | 0.00,1.00,0.00 |  0.561439 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 136ms/step\n",
      "1/1 [==============================] - 0s 140ms/step\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      51 | target_f     | 0.00,1.00,0.00 |  0.620878 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      51 | suggested    | 0.00,1.00,0.62 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      51 | final        | 0.00,0.00,1.00 |  0.603605 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 144ms/step\n",
      "1/1 [==============================] - 0s 164ms/step\n",
      "1/1 [==============================] - 0s 226ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      52 | target_f     | 0.00,0.00,1.00 |  0.609478 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      52 | suggested    | 0.00,0.61,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      52 | final        | 0.00,1.00,0.00 |  0.599323 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "1/1 [==============================] - 0s 142ms/step\n",
      "1/1 [==============================] - 0s 159ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      53 | target_f     | 0.00,1.00,0.00 |  0.611761 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      53 | suggested    | 0.00,1.00,0.61 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      53 | final        | 0.00,0.00,1.00 |  0.614092 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "1/1 [==============================] - 0s 155ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      54 | target_f     | 0.00,0.00,1.00 |  0.043475 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      54 | suggested    | 0.00,0.00,0.04 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      54 | final        | 0.00,1.00,0.00 |  0.530826 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 132ms/step\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      55 | target_f     | 0.00,1.00,0.00 |  0.705338 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      55 | suggested    | 0.00,0.71,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      55 | final        | 0.00,1.00,0.00 |  0.680225 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 141ms/step\n",
      "1/1 [==============================] - 0s 154ms/step\n",
      "1/1 [==============================] - 0s 159ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      56 | target_f     | 0.00,1.00,0.00 |  0.674001 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      56 | suggested    | 0.00,0.67,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      56 | final        | 0.00,0.00,1.00 |  0.668769 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 132ms/step\n",
      "1/1 [==============================] - 0s 144ms/step\n",
      "1/1 [==============================] - 0s 156ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      57 | target_f     | 0.00,0.00,1.00 |  0.717045 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      57 | suggested    | 0.00,0.72,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      57 | final        | 0.00,1.00,0.00 |  0.690887 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "1/1 [==============================] - 0s 144ms/step\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      58 | target_f     | 0.00,1.00,0.00 |  0.576575 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      58 | suggested    | 0.00,0.58,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      58 | final        | 0.00,1.00,0.00 |  0.602466 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 165ms/step\n",
      "1/1 [==============================] - 0s 139ms/step\n",
      "1/1 [==============================] - 0s 153ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      59 | target_f     | 0.00,1.00,0.00 |  0.595315 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      59 | suggested    | 0.00,0.60,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      59 | final        | 0.00,0.00,1.00 |  0.610224 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 157ms/step\n",
      "1/1 [==============================] - 0s 159ms/step\n",
      "1/1 [==============================] - 0s 164ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      60 | target_f     | 0.00,0.00,1.00 |  0.733832 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      60 | suggested    | 0.00,0.73,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      60 | final        | 0.00,1.00,0.00 |  0.703364 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 144ms/step\n",
      "1/1 [==============================] - 0s 139ms/step\n",
      "1/1 [==============================] - 0s 161ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      61 | target_f     | 0.00,1.00,0.00 |  0.654813 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      61 | suggested    | 0.00,1.00,0.65 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      61 | final        | 0.00,0.00,1.00 |  0.638224 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      62 | target_f     | 0.00,0.00,1.00 |  0.651783 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      62 | suggested    | 0.00,0.65,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      62 | final        | 0.00,1.00,0.00 |  0.653259 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 202ms/step\n",
      "1/1 [==============================] - 0s 140ms/step\n",
      "1/1 [==============================] - 0s 134ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      63 | target_f     | 0.00,1.00,0.00 |  0.738839 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      63 | suggested    | 0.00,0.74,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      63 | final        | 0.00,1.00,0.00 |  0.734427 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 155ms/step\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      64 | target_f     | 0.00,1.00,0.00 |  0.693432 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      64 | suggested    | 0.00,0.69,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      64 | final        | 0.00,0.00,1.00 |  0.696425 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 161ms/step\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      65 | target_f     | 0.00,0.00,1.00 |  0.599298 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      65 | suggested    | 0.00,0.00,0.60 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      65 | final        | 0.00,1.00,0.00 |  0.589692 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "1/1 [==============================] - 0s 154ms/step\n",
      "1/1 [==============================] - 0s 287ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      66 | target_f     | 0.00,1.00,0.00 |  0.745724 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      66 | suggested    | 0.00,0.75,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      66 | final        | 0.00,1.00,0.00 |  0.726818 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      67 | target_f     | 0.00,1.00,0.00 |  0.0418   |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      67 | suggested    | 0.00,1.00,0.04 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      67 | final        | 0.00,1.00,0.00 |  0.548084 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 132ms/step\n",
      "1/1 [==============================] - 0s 136ms/step\n",
      "1/1 [==============================] - 0s 139ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      68 | target_f     | 0.00,1.00,0.00 |  0.612135 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      68 | suggested    | 0.00,1.00,0.61 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      68 | final        | 0.00,0.00,1.00 |  0.560748 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 140ms/step\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      69 | target_f     | 0.00,0.00,1.00 |  0.562628 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      69 | suggested    | 0.00,0.00,0.56 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      69 | final        | 0.00,0.00,1.00 |  0.542672 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      70 | target_f     | 0.00,0.00,1.00 |  0.72859  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      70 | suggested    | 0.00,0.73,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      70 | final        | 0.00,1.00,0.00 |  0.721589 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 136ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "1/1 [==============================] - 0s 154ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      71 | target_f     | 0.00,1.00,0.00 |  0.621517 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      71 | suggested    | 0.00,0.62,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      71 | final        | 0.00,1.00,0.00 |  0.62266  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 139ms/step\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      72 | target_f     | 0.00,1.00,0.00 |  0.685974 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      72 | suggested    | 0.00,1.00,0.69 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      72 | final        | 0.00,0.00,1.00 |  0.585107 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 135ms/step\n",
      "1/1 [==============================] - 0s 157ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      73 | target_f     | 0.00,0.00,1.00 |  0.743186 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      73 | suggested    | 0.00,0.74,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      73 | final        | 0.00,1.00,0.00 |  0.737117 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 139ms/step\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "1/1 [==============================] - 0s 152ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      74 | target_f     | 0.00,1.00,0.00 |  0.717621 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      74 | suggested    | 0.00,0.72,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      74 | final        | 0.00,0.00,1.00 |  0.720089 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 155ms/step\n",
      "1/1 [==============================] - 0s 135ms/step\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      75 | target_f     | 0.00,0.00,1.00 |  0.645749 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      75 | suggested    | 0.00,0.00,0.65 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      75 | final        | 0.00,1.00,0.00 |  0.524946 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 128ms/step\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 156ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      76 | target_f     | 0.00,1.00,0.00 |  0.67744  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      76 | suggested    | 0.00,1.00,0.68 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      76 | final        | 0.00,0.00,1.00 |  0.632112 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 152ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 154ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      77 | target_f     | 0.00,0.00,1.00 |  0.702139 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      77 | suggested    | 0.00,0.00,0.70 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      77 | final        | 0.00,0.00,1.00 |  0.640084 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 153ms/step\n",
      "1/1 [==============================] - 0s 133ms/step\n",
      "1/1 [==============================] - 0s 145ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      78 | target_f     | 0.00,0.00,1.00 |  0.592588 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      78 | suggested    | 0.00,0.59,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      78 | final        | 0.00,1.00,0.00 |  0.615171 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 158ms/step\n",
      "1/1 [==============================] - 0s 151ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      79 | target_f     | 0.00,1.00,0.00 |  0.092425 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      79 | suggested    | 0.00,1.00,0.09 |  2        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      79 | final        | 0.00,1.00,0.00 |  0.512928 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 132ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "1/1 [==============================] - 0s 153ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      80 | target_f     | 0.00,1.00,0.00 |  0.634266 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      80 | suggested    | 0.00,0.63,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      80 | final        | 0.00,0.00,1.00 |  0.634603 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      81 | target_f     | 0.00,0.00,1.00 |  0.592886 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      81 | suggested    | 0.00,0.59,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      81 | final        | 0.00,1.00,0.00 |  0.595129 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 150ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      82 | target_f     | 0.00,1.00,0.00 |  0.590072 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      82 | suggested    | 0.00,0.59,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      82 | final        | 0.00,1.00,0.00 |  0.595129 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 147ms/step\n",
      "1/1 [==============================] - 0s 158ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      83 | target_f     | 0.00,1.00,0.00 |  0.658406 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      83 | suggested    | 0.00,0.66,0.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      83 | final        | 0.00,0.00,1.00 |  0.65179  |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 148ms/step\n",
      "1/1 [==============================] - 0s 149ms/step\n",
      "1/1 [==============================] - 0s 136ms/step\n",
      "+---------+--------------+----------------+-----------+\n",
      "|   index | Variations   | Probs          |   rewards |\n",
      "+=========+==============+================+===========+\n",
      "|      84 | target_f     | 0.00,0.00,1.00 |  0.676962 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      84 | suggested    | 0.00,0.68,1.00 |  1        |\n",
      "+---------+--------------+----------------+-----------+\n",
      "|      84 | final        | 0.00,1.00,0.00 |  0.672545 |\n",
      "+---------+--------------+----------------+-----------+\n",
      "1/1 [==============================] - 0s 146ms/step\n",
      "1/1 [==============================] - 0s 138ms/step\n"
     ]
    },
    {
     "ename": "ProgrammingError",
     "evalue": "Cannot operate on a closed database.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/framework/ops.py:2698\u001b[0m, in \u001b[0;36mOperation.get_attr\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   2697\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m c_api_util\u001b[38;5;241m.\u001b[39mtf_buffer() \u001b[38;5;28;01mas\u001b[39;00m buf:\n\u001b[0;32m-> 2698\u001b[0m   \u001b[43mpywrap_tf_session\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTF_OperationGetAttrValueProto\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_c_op\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuf\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   2699\u001b[0m   data \u001b[38;5;241m=\u001b[39m pywrap_tf_session\u001b[38;5;241m.\u001b[39mTF_GetBuffer(buf)\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: Operation 'TensorSliceDataset' has no attr named '_read_only_resource_inputs'.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[49], line 180\u001b[0m, in \u001b[0;36mAgent.replay_v3\u001b[0;34m(self, batch_size, generate_dataset)\u001b[0m\n\u001b[1;32m    178\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcritic_model\u001b[38;5;241m.\u001b[39msave_weights(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_directory \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_name \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m\u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmemory_indexer)\u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m_critic.h5\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 180\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mactor_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mstate\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtarget_f\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m1\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mverbose\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m    181\u001b[0m target_critic \u001b[38;5;241m=\u001b[39m np\u001b[38;5;241m.\u001b[39marray([[target]])\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/keras/utils/traceback_utils.py:65\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     64\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 65\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     66\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/keras/engine/training.py:1625\u001b[0m, in \u001b[0;36mModel.fit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1621\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdistribute_strategy\u001b[38;5;241m.\u001b[39mscope(), training_utils\u001b[38;5;241m.\u001b[39mRespectCompiledTrainableState(  \u001b[38;5;66;03m# noqa: E501\u001b[39;00m\n\u001b[1;32m   1622\u001b[0m     \u001b[38;5;28mself\u001b[39m\n\u001b[1;32m   1623\u001b[0m ):\n\u001b[1;32m   1624\u001b[0m     \u001b[38;5;66;03m# Creates a `tf.data.Dataset` and handles batch and epoch iteration.\u001b[39;00m\n\u001b[0;32m-> 1625\u001b[0m     data_handler \u001b[38;5;241m=\u001b[39m \u001b[43mdata_adapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_data_handler\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1626\u001b[0m \u001b[43m        \u001b[49m\u001b[43mx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1627\u001b[0m \u001b[43m        \u001b[49m\u001b[43my\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1628\u001b[0m \u001b[43m        \u001b[49m\u001b[43msample_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1629\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1630\u001b[0m \u001b[43m        \u001b[49m\u001b[43msteps_per_epoch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msteps_per_epoch\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1631\u001b[0m \u001b[43m        \u001b[49m\u001b[43minitial_epoch\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minitial_epoch\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1632\u001b[0m \u001b[43m        \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mepochs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1633\u001b[0m \u001b[43m        \u001b[49m\u001b[43mshuffle\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshuffle\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1634\u001b[0m \u001b[43m        \u001b[49m\u001b[43mclass_weight\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mclass_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1635\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmax_queue_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmax_queue_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1636\u001b[0m \u001b[43m        \u001b[49m\u001b[43mworkers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mworkers\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1637\u001b[0m \u001b[43m        \u001b[49m\u001b[43muse_multiprocessing\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_multiprocessing\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1638\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1639\u001b[0m \u001b[43m        \u001b[49m\u001b[43msteps_per_execution\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_steps_per_execution\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1640\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1642\u001b[0m     \u001b[38;5;66;03m# Container that configures and calls `tf.keras.Callback`s.\u001b[39;00m\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/keras/engine/data_adapter.py:1583\u001b[0m, in \u001b[0;36mget_data_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m   1582\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m _ClusterCoordinatorDataHandler(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m-> 1583\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mDataHandler\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/keras/engine/data_adapter.py:1260\u001b[0m, in \u001b[0;36mDataHandler.__init__\u001b[0;34m(self, x, y, sample_weight, batch_size, steps_per_epoch, initial_epoch, epochs, shuffle, class_weight, max_queue_size, workers, use_multiprocessing, model, steps_per_execution, distribute)\u001b[0m\n\u001b[1;32m   1259\u001b[0m adapter_cls \u001b[38;5;241m=\u001b[39m select_data_adapter(x, y)\n\u001b[0;32m-> 1260\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_adapter \u001b[38;5;241m=\u001b[39m \u001b[43madapter_cls\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1261\u001b[0m \u001b[43m    \u001b[49m\u001b[43mx\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1262\u001b[0m \u001b[43m    \u001b[49m\u001b[43my\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1263\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1264\u001b[0m \u001b[43m    \u001b[49m\u001b[43msteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msteps_per_epoch\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1265\u001b[0m \u001b[43m    \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mepochs\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m-\u001b[39;49m\u001b[43m \u001b[49m\u001b[43minitial_epoch\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1266\u001b[0m \u001b[43m    \u001b[49m\u001b[43msample_weights\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msample_weight\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1267\u001b[0m \u001b[43m    \u001b[49m\u001b[43mshuffle\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshuffle\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1268\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmax_queue_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmax_queue_size\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1269\u001b[0m \u001b[43m    \u001b[49m\u001b[43mworkers\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mworkers\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1270\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_multiprocessing\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_multiprocessing\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1271\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdistribution_strategy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtf\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdistribute\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_strategy\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1272\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1273\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1275\u001b[0m strategy \u001b[38;5;241m=\u001b[39m tf\u001b[38;5;241m.\u001b[39mdistribute\u001b[38;5;241m.\u001b[39mget_strategy()\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/keras/engine/data_adapter.py:346\u001b[0m, in \u001b[0;36mTensorLikeDataAdapter.__init__\u001b[0;34m(self, x, y, sample_weights, sample_weight_modes, batch_size, epochs, steps, shuffle, **kwargs)\u001b[0m\n\u001b[1;32m    344\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m flat_dataset\n\u001b[0;32m--> 346\u001b[0m indices_dataset \u001b[38;5;241m=\u001b[39m \u001b[43mindices_dataset\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mflat_map\u001b[49m\u001b[43m(\u001b[49m\u001b[43mslice_batch_indices\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    348\u001b[0m dataset \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mslice_inputs(indices_dataset, inputs)\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/data/ops/dataset_ops.py:2285\u001b[0m, in \u001b[0;36mDatasetV2.flat_map\u001b[0;34m(self, map_func, name)\u001b[0m\n\u001b[1;32m   2284\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtensorflow\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mpython\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdata\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mops\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m flat_map_op\n\u001b[0;32m-> 2285\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mflat_map_op\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_flat_map\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmap_func\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/data/ops/flat_map_op.py:24\u001b[0m, in \u001b[0;36m_flat_map\u001b[0;34m(input_dataset, map_func, name)\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"See `Dataset.flat_map()` for details.\"\"\"\u001b[39;00m\n\u001b[0;32m---> 24\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_FlatMapDataset\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_dataset\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmap_func\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/data/ops/flat_map_op.py:33\u001b[0m, in \u001b[0;36m_FlatMapDataset.__init__\u001b[0;34m(self, input_dataset, map_func, name)\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_input_dataset \u001b[38;5;241m=\u001b[39m input_dataset\n\u001b[0;32m---> 33\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_map_func \u001b[38;5;241m=\u001b[39m \u001b[43mstructured_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mStructuredFunctionWrapper\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     34\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmap_func\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_transformation_name\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdataset\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minput_dataset\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     35\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_map_func\u001b[38;5;241m.\u001b[39moutput_structure, dataset_ops\u001b[38;5;241m.\u001b[39mDatasetSpec):\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/data/ops/structured_function.py:261\u001b[0m, in \u001b[0;36mStructuredFunctionWrapper.__init__\u001b[0;34m(self, func, transformation_name, dataset, input_classes, input_shapes, input_types, input_structure, add_to_graph, use_legacy_function, defun_kwargs)\u001b[0m\n\u001b[1;32m    259\u001b[0m     fn_factory \u001b[38;5;241m=\u001b[39m trace_tf_function(defun_kwargs)\n\u001b[0;32m--> 261\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_function \u001b[38;5;241m=\u001b[39m \u001b[43mfn_factory\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    262\u001b[0m \u001b[38;5;66;03m# There is no graph to add in eager mode.\u001b[39;00m\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/eager/polymorphic_function/tracing_compiler.py:232\u001b[0m, in \u001b[0;36mTracingCompiler.get_concrete_function\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    224\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Returns a `ConcreteFunction` specialized to inputs and execution context.\u001b[39;00m\n\u001b[1;32m    225\u001b[0m \n\u001b[1;32m    226\u001b[0m \u001b[38;5;124;03mArgs:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    230\u001b[0m \u001b[38;5;124;03m    `tf.Tensor` or `tf.TensorSpec`.\u001b[39;00m\n\u001b[1;32m    231\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 232\u001b[0m concrete_function \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_concrete_function_garbage_collected\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    233\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    234\u001b[0m concrete_function\u001b[38;5;241m.\u001b[39m_garbage_collector\u001b[38;5;241m.\u001b[39mrelease()  \u001b[38;5;66;03m# pylint: disable=protected-access\u001b[39;00m\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/eager/polymorphic_function/tracing_compiler.py:202\u001b[0m, in \u001b[0;36mTracingCompiler._get_concrete_function_garbage_collected\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    201\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[0;32m--> 202\u001b[0m   concrete_function, _ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_maybe_define_concrete_function\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    203\u001b[0m   seen_names \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/eager/polymorphic_function/tracing_compiler.py:166\u001b[0m, in \u001b[0;36mTracingCompiler._maybe_define_concrete_function\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m    164\u001b[0m   kwargs \u001b[38;5;241m=\u001b[39m {}\n\u001b[0;32m--> 166\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_maybe_define_function\u001b[49m\u001b[43m(\u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/eager/polymorphic_function/tracing_compiler.py:396\u001b[0m, in \u001b[0;36mTracingCompiler._maybe_define_function\u001b[0;34m(self, args, kwargs)\u001b[0m\n\u001b[1;32m    394\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m placeholder_bound_args\u001b[38;5;241m.\u001b[39mkwargs\n\u001b[0;32m--> 396\u001b[0m concrete_function \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_create_concrete_function\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    397\u001b[0m \u001b[43m    \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mfunc_graph\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    399\u001b[0m \u001b[38;5;66;03m# TODO(b/263520817): Remove access to private attribute.\u001b[39;00m\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/eager/polymorphic_function/tracing_compiler.py:300\u001b[0m, in \u001b[0;36mTracingCompiler._create_concrete_function\u001b[0;34m(self, args, kwargs, func_graph)\u001b[0m\n\u001b[1;32m    297\u001b[0m   arg_names \u001b[38;5;241m=\u001b[39m base_arg_names\n\u001b[1;32m    299\u001b[0m concrete_function \u001b[38;5;241m=\u001b[39m monomorphic_function\u001b[38;5;241m.\u001b[39mConcreteFunction(\n\u001b[0;32m--> 300\u001b[0m     \u001b[43mfunc_graph_module\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfunc_graph_from_py_func\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    301\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    302\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_python_function\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    303\u001b[0m \u001b[43m        \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    304\u001b[0m \u001b[43m        \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    305\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43;01mNone\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    306\u001b[0m \u001b[43m        \u001b[49m\u001b[43mfunc_graph\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfunc_graph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    307\u001b[0m \u001b[43m        \u001b[49m\u001b[43mautograph\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_autograph\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    308\u001b[0m \u001b[43m        \u001b[49m\u001b[43mautograph_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_autograph_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    309\u001b[0m \u001b[43m        \u001b[49m\u001b[43marg_names\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43marg_names\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    310\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcapture_by_value\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_capture_by_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    311\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcreate_placeholders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m,\n\u001b[1;32m    312\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_function_attributes,\n\u001b[1;32m    313\u001b[0m     spec\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfunction_spec,\n\u001b[1;32m    314\u001b[0m     \u001b[38;5;66;03m# Tell the ConcreteFunction to clean up its graph once it goes out of\u001b[39;00m\n\u001b[1;32m    315\u001b[0m     \u001b[38;5;66;03m# scope. This is not the default behavior since it gets used in some\u001b[39;00m\n\u001b[1;32m    316\u001b[0m     \u001b[38;5;66;03m# places (like Keras) where the FuncGraph lives longer than the\u001b[39;00m\n\u001b[1;32m    317\u001b[0m     \u001b[38;5;66;03m# ConcreteFunction.\u001b[39;00m\n\u001b[1;32m    318\u001b[0m     shared_func_graph\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m    319\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m concrete_function\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/framework/func_graph.py:1113\u001b[0m, in \u001b[0;36mfunc_graph_from_py_func\u001b[0;34m(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, create_placeholders, acd_record_initial_resource_uses)\u001b[0m\n\u001b[1;32m   1111\u001b[0m   deps_control_manager \u001b[38;5;241m=\u001b[39m ops\u001b[38;5;241m.\u001b[39mNullContextmanager()\n\u001b[0;32m-> 1113\u001b[0m \u001b[43m\u001b[49m\u001b[38;5;28;43;01mwith\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mfunc_graph\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mas_default\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdeps_control_manager\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mas\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mdeps_ctx\u001b[49m\u001b[43m:\u001b[49m\n\u001b[1;32m   1114\u001b[0m \u001b[43m  \u001b[49m\u001b[43mcurrent_scope\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mvariable_scope\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_variable_scope\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/framework/auto_control_deps.py:465\u001b[0m, in \u001b[0;36mAutomaticControlDependencies.__exit__\u001b[0;34m(self, unused_type, unused_value, unused_traceback)\u001b[0m\n\u001b[1;32m    463\u001b[0m \u001b[38;5;66;03m# Check for any resource inputs. If we find any, we update control_inputs\u001b[39;00m\n\u001b[1;32m    464\u001b[0m \u001b[38;5;66;03m# and last_write_to_resource.\u001b[39;00m\n\u001b[0;32m--> 465\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m inp, resource_type \u001b[38;5;129;01min\u001b[39;00m _get_resource_inputs(op):\n\u001b[1;32m    466\u001b[0m   is_read \u001b[38;5;241m=\u001b[39m resource_type \u001b[38;5;241m==\u001b[39m ResourceType\u001b[38;5;241m.\u001b[39mREAD_ONLY\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/framework/auto_control_deps.py:663\u001b[0m, in \u001b[0;36m_get_resource_inputs\u001b[0;34m(op)\u001b[0m\n\u001b[1;32m    662\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Returns an iterable of resources touched by this `op`.\"\"\"\u001b[39;00m\n\u001b[0;32m--> 663\u001b[0m reads, writes \u001b[38;5;241m=\u001b[39m \u001b[43mutils\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_read_write_resource_inputs\u001b[49m\u001b[43m(\u001b[49m\u001b[43mop\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    664\u001b[0m saturated \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/framework/auto_control_deps_utils.py:105\u001b[0m, in \u001b[0;36mget_read_write_resource_inputs\u001b[0;34m(op)\u001b[0m\n\u001b[1;32m    104\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 105\u001b[0m   read_only_input_indices \u001b[38;5;241m=\u001b[39m \u001b[43mop\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_attr\u001b[49m\u001b[43m(\u001b[49m\u001b[43mREAD_ONLY_RESOURCE_INPUTS_ATTR\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    106\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m:\n\u001b[1;32m    107\u001b[0m   \u001b[38;5;66;03m# Attr was not set. Add all resource inputs to `writes` and return.\u001b[39;00m\n",
      "File \u001b[0;32m~/Projects/ForexLab/dataset/episode2/env/lib/python3.11/site-packages/tensorflow/python/framework/ops.py:2697\u001b[0m, in \u001b[0;36mOperation.get_attr\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m   2696\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 2697\u001b[0m \u001b[43m  \u001b[49m\u001b[38;5;28;43;01mwith\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mc_api_util\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtf_buffer\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mas\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mbuf\u001b[49m\u001b[43m:\u001b[49m\n\u001b[1;32m   2698\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpywrap_tf_session\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTF_OperationGetAttrValueProto\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_c_op\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbuf\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/usr/lib/python3.11/contextlib.py:141\u001b[0m, in \u001b[0;36m_GeneratorContextManager.__exit__\u001b[0;34m(self, typ, value, traceback)\u001b[0m\n\u001b[1;32m    139\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mgenerator didn\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt yield\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 141\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__exit__\u001b[39m(\u001b[38;5;28mself\u001b[39m, typ, value, traceback):\n\u001b[1;32m    142\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m typ \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: ",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mProgrammingError\u001b[0m                          Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[56], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43magent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mreplay_v3\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m32\u001b[39;49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[0;32mIn[49], line 211\u001b[0m, in \u001b[0;36mAgent.replay_v3\u001b[0;34m(self, batch_size, generate_dataset)\u001b[0m\n\u001b[1;32m    208\u001b[0m         agent\u001b[38;5;241m.\u001b[39mepsilon \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m=\u001b[39m agent\u001b[38;5;241m.\u001b[39mepsilon_decay\n\u001b[1;32m    210\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mKeyboardInterrupt\u001b[39;00m:\n\u001b[0;32m--> 211\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshutdown\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    212\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msave_model()\n\u001b[1;32m    213\u001b[0m     time\u001b[38;5;241m.\u001b[39msleep(\u001b[38;5;241m10\u001b[39m)\n",
      "Cell \u001b[0;32mIn[49], line 76\u001b[0m, in \u001b[0;36mAgent.shutdown\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     75\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mshutdown\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m---> 76\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcursor\u001b[38;5;241m.\u001b[39mclose()\n\u001b[1;32m     77\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconnection\u001b[38;5;241m.\u001b[39mclose()\n",
      "\u001b[0;31mProgrammingError\u001b[0m: Cannot operate on a closed database."
     ]
    }
   ],
   "source": [
    "agent.replay_v3(32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "gpuClass": "standard",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}