mariotut.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "mount_file_id": "1W7AMeb7KKADG6U3g5VBAQvbj4C2Z1UYG",
      "authorship_tag": "ABX9TyOw84CU2E0MOUjTnkH2J4Tg",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yashgosa/my_ml_projects/blob/main/mariotut.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "X1KjXgyf6nk-",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 328
        },
        "outputId": "a7abffda-0c35-4d18-f4f3-3d48a59f48c5"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "error",
          "ename": "MessageError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mMessageError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-d5df0069828e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolab\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdrive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mdrive\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/google/colab/drive.py\u001b[0m in \u001b[0;36mmount\u001b[0;34m(mountpoint, force_remount, timeout_ms, readonly)\u001b[0m\n\u001b[1;32m    104\u001b[0m       \u001b[0mtimeout_ms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout_ms\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m       \u001b[0mephemeral\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m       readonly=readonly)\n\u001b[0m\u001b[1;32m    107\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/google/colab/drive.py\u001b[0m in \u001b[0;36m_mount\u001b[0;34m(mountpoint, force_remount, timeout_ms, ephemeral, readonly)\u001b[0m\n\u001b[1;32m    123\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mephemeral\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m     _message.blocking_request(\n\u001b[0;32m--> 125\u001b[0;31m         'request_auth', request={'authType': 'dfs_ephemeral'}, timeout_sec=None)\n\u001b[0m\u001b[1;32m    126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m   \u001b[0mmountpoint\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_os\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexpanduser\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmountpoint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/google/colab/_message.py\u001b[0m in \u001b[0;36mblocking_request\u001b[0;34m(request_type, request, timeout_sec, parent)\u001b[0m\n\u001b[1;32m    169\u001b[0m   request_id = send_request(\n\u001b[1;32m    170\u001b[0m       request_type, request, parent=parent, expect_reply=True)\n\u001b[0;32m--> 171\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mread_reply_from_input\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtimeout_sec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/google/colab/_message.py\u001b[0m in \u001b[0;36mread_reply_from_input\u001b[0;34m(message_id, timeout_sec)\u001b[0m\n\u001b[1;32m    100\u001b[0m         reply.get('colab_msg_id') == message_id):\n\u001b[1;32m    101\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;34m'error'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mreply\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 102\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mMessageError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreply\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'error'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    103\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mreply\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'data'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mMessageError\u001b[0m: Error: credential propagation was unsuccessful"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gym-super-mario-bros"
      ],
      "metadata": {
        "id": "NhNN86P14aGz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c66cc8a9-8c1b-4e81-8e2c-678c425fbf2c"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting gym-super-mario-bros\n",
            "  Downloading gym_super_mario_bros-7.4.0-py3-none-any.whl (199 kB)\n",
            "\u001b[K     |████████████████████████████████| 199 kB 26.9 MB/s \n",
            "\u001b[?25hCollecting nes-py>=8.1.4\n",
            "  Downloading nes_py-8.2.1.tar.gz (77 kB)\n",
            "\u001b[K     |████████████████████████████████| 77 kB 6.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: gym>=0.17.2 in /usr/local/lib/python3.7/dist-packages (from nes-py>=8.1.4->gym-super-mario-bros) (0.25.2)\n",
            "Requirement already satisfied: numpy>=1.18.5 in /usr/local/lib/python3.7/dist-packages (from nes-py>=8.1.4->gym-super-mario-bros) (1.21.6)\n",
            "Collecting pyglet<=1.5.21,>=1.4.0\n",
            "  Downloading pyglet-1.5.21-py3-none-any.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 14.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.48.2 in /usr/local/lib/python3.7/dist-packages (from nes-py>=8.1.4->gym-super-mario-bros) (4.64.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.8.0 in /usr/local/lib/python3.7/dist-packages (from gym>=0.17.2->nes-py>=8.1.4->gym-super-mario-bros) (4.13.0)\n",
            "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.7/dist-packages (from gym>=0.17.2->nes-py>=8.1.4->gym-super-mario-bros) (0.0.8)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym>=0.17.2->nes-py>=8.1.4->gym-super-mario-bros) (1.5.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.8.0->gym>=0.17.2->nes-py>=8.1.4->gym-super-mario-bros) (3.10.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.8.0->gym>=0.17.2->nes-py>=8.1.4->gym-super-mario-bros) (4.1.1)\n",
            "Building wheels for collected packages: nes-py\n",
            "  Building wheel for nes-py (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for nes-py: filename=nes_py-8.2.1-cp37-cp37m-linux_x86_64.whl size=434766 sha256=44640930d9ce1bd26bef9a00d81b6ec5989c54210860670ce9910051fec05c06\n",
            "  Stored in directory: /root/.cache/pip/wheels/17/96/0e/22a8c7dbdf412d8e988286f223b223baf0f4ad90c9e699c56d\n",
            "Successfully built nes-py\n",
            "Installing collected packages: pyglet, nes-py, gym-super-mario-bros\n",
            "Successfully installed gym-super-mario-bros-7.4.0 nes-py-8.2.1 pyglet-1.5.21\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install nespy"
      ],
      "metadata": {
        "id": "m2jSuPMh4aOB",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8f925cd4-0568-42f2-cfdd-cf438bf38b76"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting nespy\n",
            "  Downloading nespy-0.0.6-py3-none-any.whl (53 kB)\n",
            "\u001b[K     |████████████████████████████████| 53 kB 1.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: ipython>=6 in /usr/local/lib/python3.7/dist-packages (from nespy) (7.9.0)\n",
            "Collecting requests>=2.25\n",
            "  Downloading requests-2.28.1-py3-none-any.whl (62 kB)\n",
            "\u001b[K     |████████████████████████████████| 62 kB 1.6 MB/s \n",
            "\u001b[?25hCollecting google>=3\n",
            "  Downloading google-3.0.0-py2.py3-none-any.whl (45 kB)\n",
            "\u001b[K     |████████████████████████████████| 45 kB 3.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3 in /usr/local/lib/python3.7/dist-packages (from nespy) (3.19.6)\n",
            "Requirement already satisfied: pyzmq>=22 in /usr/local/lib/python3.7/dist-packages (from nespy) (23.2.1)\n",
            "Requirement already satisfied: pandas>=1 in /usr/local/lib/python3.7/dist-packages (from nespy) (1.3.5)\n",
            "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.7/dist-packages (from google>=3->nespy) (4.6.3)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipython>=6->nespy) (5.1.1)\n",
            "Collecting jedi>=0.10\n",
            "  Downloading jedi-0.18.1-py2.py3-none-any.whl (1.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.6 MB 70.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=6->nespy) (2.6.1)\n",
            "Requirement already satisfied: pexpect in /usr/local/lib/python3.7/dist-packages (from ipython>=6->nespy) (4.8.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.1.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from ipython>=6->nespy) (2.0.10)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython>=6->nespy) (4.4.2)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.7/dist-packages (from ipython>=6->nespy) (0.2.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=6->nespy) (0.7.5)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython>=6->nespy) (57.4.0)\n",
            "Requirement already satisfied: parso<0.9.0,>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from jedi>=0.10->ipython>=6->nespy) (0.8.3)\n",
            "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1->nespy) (1.21.6)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1->nespy) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=1->nespy) (2022.6)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython>=6->nespy) (1.15.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython>=6->nespy) (0.2.5)\n",
            "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.25->nespy) (2.1.1)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.25->nespy) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.25->nespy) (2022.9.24)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.25->nespy) (1.24.3)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect->ipython>=6->nespy) (0.7.0)\n",
            "Installing collected packages: jedi, requests, google, nespy\n",
            "  Attempting uninstall: requests\n",
            "    Found existing installation: requests 2.23.0\n",
            "    Uninstalling requests-2.23.0:\n",
            "      Successfully uninstalled requests-2.23.0\n",
            "  Attempting uninstall: google\n",
            "    Found existing installation: google 2.0.3\n",
            "    Uninstalling google-2.0.3:\n",
            "      Successfully uninstalled google-2.0.3\n",
            "Successfully installed google-3.0.0 jedi-0.18.1 nespy-0.0.6 requests-2.28.1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install nes_py"
      ],
      "metadata": {
        "id": "rGr6kTbP5sru",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a1e9a733-baec-4dd3-b73f-ecb3d7bb5875"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: nes_py in /usr/local/lib/python3.7/dist-packages (8.2.1)\n",
            "Requirement already satisfied: numpy>=1.18.5 in /usr/local/lib/python3.7/dist-packages (from nes_py) (1.21.6)\n",
            "Requirement already satisfied: tqdm>=4.48.2 in /usr/local/lib/python3.7/dist-packages (from nes_py) (4.64.1)\n",
            "Requirement already satisfied: pyglet<=1.5.21,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from nes_py) (1.5.21)\n",
            "Requirement already satisfied: gym>=0.17.2 in /usr/local/lib/python3.7/dist-packages (from nes_py) (0.25.2)\n",
            "Requirement already satisfied: gym-notices>=0.0.4 in /usr/local/lib/python3.7/dist-packages (from gym>=0.17.2->nes_py) (0.0.8)\n",
            "Requirement already satisfied: cloudpickle>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym>=0.17.2->nes_py) (1.5.0)\n",
            "Requirement already satisfied: importlib-metadata>=4.8.0 in /usr/local/lib/python3.7/dist-packages (from gym>=0.17.2->nes_py) (4.13.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.8.0->gym>=0.17.2->nes_py) (4.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.8.0->gym>=0.17.2->nes_py) (3.10.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#TIP: \n",
        "WHENEVER I AM TRYING TO AUTOMATE A GAME TRY TO AIMPLIFY THE ENVOIRMENT MAKING IT COMPUTATIONALLY LIGHTER FOR THE SYSTEM AND MAKING IT EASY TO BE LEARNED"
      ],
      "metadata": {
        "id": "CtjePC326Qbg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#Importing the game\n",
        "import gym_super_mario_bros \n",
        "#Importing the joypad wrapper\n",
        "from nes_py.wrappers import JoypadSpace\n",
        "#Impirt SIMPLIFIED Controls\n",
        "from gym_super_mario_bros.actions import SIMPLE_MOVEMENT"
      ],
      "metadata": {
        "id": "SlqVXzGe4aSu"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#setup game\n",
        "env = gym_super_mario_bros.make(\"SuperMarioBros-v3\")  # v0 is outadated the latest is v3\n",
        "env = JoypadSpace(env, SIMPLE_MOVEMENT)"
      ],
      "metadata": {
        "id": "2L5Wh3Q_6AbQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "39cb0a12-fef3-4e3a-8f33-44bb3c2930eb"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/gym/core.py:318: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  \"Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\"\n",
            "/usr/local/lib/python3.7/dist-packages/gym/wrappers/step_api_compatibility.py:40: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  \"Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# a flag for starting the game\n",
        "done = True\n",
        "#Looping through 10000 frames of the game\n",
        "for step in range(10000):\n",
        "  #Check if we are done playing the game\n",
        "  if done:\n",
        "    #Restarts the game to begin with\n",
        "    env.reset()\n",
        "  #Do random actions\n",
        "  state, reward, done, info = env.step(env.action_space.sample())\n",
        "  #Display the game\n",
        "  env.render()\n",
        "#Close the game\n",
        "env.close()"
      ],
      "metadata": {
        "id": "IVwrU0MQ7ip7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 556
        },
        "outputId": "76a6114e-947b-43d2-cc4d-4041d7339dca"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/gym/utils/passive_env_checker.py:228: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n",
            "/usr/local/lib/python3.7/dist-packages/gym/core.py:50: DeprecationWarning: \u001b[33mWARN: You are calling render method, but you didn't specified the argument render_mode at environment initialization. To maintain backward compatibility, the environment will render in human mode.\n",
            "If you want to render in human mode, initialize the environment in this way: gym.make('EnvName', render_mode='human') and don't call the render method.\n",
            "See here for more information: https://www.gymlibrary.ml/content/api/\u001b[0m\n",
            "  \"You are calling render method, \"\n",
            "/usr/local/lib/python3.7/dist-packages/gym/utils/passive_env_checker.py:281: UserWarning: \u001b[33mWARN: No render modes was declared in the environment (env.metadata['render_modes'] is None or not defined), you may have trouble when calling `.render()`.\u001b[0m\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "NoSuchDisplayException",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyglet/__init__.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    333\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 334\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_module\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    335\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'key'",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mNoSuchDisplayException\u001b[0m                    Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-21-a9f009ff924f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     10\u001b[0m   \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m   \u001b[0;31m#Display the game\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m   \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m \u001b[0;31m#Close the game\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/core.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     56\u001b[0m             \u001b[0minfo\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcontains\u001b[0m \u001b[0mauxiliary\u001b[0m \u001b[0mdiagnostic\u001b[0m \u001b[0minformation\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhelpful\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdebugging\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0msometimes\u001b[0m \u001b[0mlearning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \"\"\"\n\u001b[0;32m---> 58\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/core.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/core.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     56\u001b[0m             \u001b[0minfo\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcontains\u001b[0m \u001b[0mauxiliary\u001b[0m \u001b[0mdiagnostic\u001b[0m \u001b[0minformation\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhelpful\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdebugging\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0msometimes\u001b[0m \u001b[0mlearning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \"\"\"\n\u001b[0;32m---> 58\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/core.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/core.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     56\u001b[0m             \u001b[0minfo\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcontains\u001b[0m \u001b[0mauxiliary\u001b[0m \u001b[0mdiagnostic\u001b[0m \u001b[0minformation\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhelpful\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdebugging\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0msometimes\u001b[0m \u001b[0mlearning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \"\"\"\n\u001b[0;32m---> 58\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/wrappers/order_enforcing.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/core.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     56\u001b[0m             \u001b[0minfo\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcontains\u001b[0m \u001b[0mauxiliary\u001b[0m \u001b[0mdiagnostic\u001b[0m \u001b[0minformation\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhelpful\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdebugging\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0msometimes\u001b[0m \u001b[0mlearning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \"\"\"\n\u001b[0;32m---> 58\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/core.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/core.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     56\u001b[0m             \u001b[0minfo\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcontains\u001b[0m \u001b[0mauxiliary\u001b[0m \u001b[0mdiagnostic\u001b[0m \u001b[0minformation\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhelpful\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdebugging\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0msometimes\u001b[0m \u001b[0mlearning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \"\"\"\n\u001b[0;32m---> 58\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/wrappers/env_checker.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/utils/passive_env_checker.py\u001b[0m in \u001b[0;36menv_render_passive_checker\u001b[0;34m(env, *args, **kwargs)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/gym/core.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m     56\u001b[0m             \u001b[0minfo\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcontains\u001b[0m \u001b[0mauxiliary\u001b[0m \u001b[0mdiagnostic\u001b[0m \u001b[0minformation\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhelpful\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdebugging\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0msometimes\u001b[0m \u001b[0mlearning\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \"\"\"\n\u001b[0;32m---> 58\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mNotImplementedError\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mabstractmethod\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/nes_py/nes_env.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, mode)\u001b[0m\n\u001b[1;32m    381\u001b[0m                     \u001b[0mcaption\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcaption\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    382\u001b[0m                     \u001b[0mheight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mSCREEN_HEIGHT\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 383\u001b[0;31m                     \u001b[0mwidth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mSCREEN_WIDTH\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    384\u001b[0m                 )\n\u001b[1;32m    385\u001b[0m             \u001b[0;31m# show the screen on the image viewer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/nes_py/_image_viewer.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, caption, height, width, monitor_keyboard, relevant_keys)\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0;31m# a mapping from pyglet key identifiers to native identifiers\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m         self.KEY_MAP = {\n\u001b[0;32m---> 35\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyglet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwindow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mENTER\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mord\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\\r'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpyglet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwindow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSPACE\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mord\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m         }\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyglet/__init__.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    338\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    339\u001b[0m             \u001b[0mimport_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'pyglet.%s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_module_name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 340\u001b[0;31m             \u001b[0m__import__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimport_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    341\u001b[0m             \u001b[0mmodule\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mimport_name\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    342\u001b[0m             \u001b[0mobject\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_module'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyglet/window/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m   1916\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0m_is_pyglet_doc_run\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1917\u001b[0m     \u001b[0mpyglet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwindow\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodules\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1918\u001b[0;31m     \u001b[0mgl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_create_shadow_window\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyglet/gl/__init__.py\u001b[0m in \u001b[0;36m_create_shadow_window\u001b[0;34m()\u001b[0m\n\u001b[1;32m    204\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m     \u001b[0;32mfrom\u001b[0m \u001b[0mpyglet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwindow\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mWindow\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 206\u001b[0;31m     \u001b[0m_shadow_window\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mWindow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwidth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvisible\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    207\u001b[0m     \u001b[0m_shadow_window\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswitch_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyglet/window/xlib/__init__.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    169\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_event_handlers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 171\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXlibWindow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    172\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    173\u001b[0m         \u001b[0;32mglobal\u001b[0m \u001b[0m_can_detect_autorepeat\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyglet/window/__init__.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, width, height, caption, resizable, style, fullscreen, visible, vsync, file_drops, display, screen, config, context, mode)\u001b[0m\n\u001b[1;32m    588\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    589\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mdisplay\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 590\u001b[0;31m             \u001b[0mdisplay\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpyglet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcanvas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_display\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    591\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    592\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mscreen\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyglet/canvas/__init__.py\u001b[0m in \u001b[0;36mget_display\u001b[0;34m()\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m     \u001b[0;31m# Otherwise, create a new display and return it.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mDisplay\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pyglet/canvas/xlib.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, name, x_screen)\u001b[0m\n\u001b[1;32m    121\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_display\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mXOpenDisplay\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    122\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_display\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 123\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mNoSuchDisplayException\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Cannot connect to \"%s\"'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m         \u001b[0mscreen_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mxlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mXScreenCount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_display\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNoSuchDisplayException\u001b[0m: Cannot connect to \"None\""
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install stable-baselines3[extra]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "4nPdpzo2bZLq",
        "outputId": "1c4393c8-a1ae-4fd8-d8a6-1bf3fea61e0a"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting stable-baselines3[extra]\n",
            "  Downloading stable_baselines3-1.6.2-py3-none-any.whl (170 kB)\n",
            "\u001b[K     |████████████████████████████████| 170 kB 34.3 MB/s \n",
            "\u001b[?25hCollecting gym==0.21\n",
            "  Downloading gym-0.21.0.tar.gz (1.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.5 MB 57.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.5.0)\n",
            "Requirement already satisfied: importlib-metadata~=4.13 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (4.13.0)\n",
            "Requirement already satisfied: torch>=1.11 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.12.1+cu113)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.3.5)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (1.21.6)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (3.2.2)\n",
            "Collecting autorom[accept-rom-license]~=0.4.2\n",
            "  Downloading AutoROM-0.4.2-py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: tensorboard>=2.9.1 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (2.9.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (5.4.8)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (4.64.1)\n",
            "Collecting ale-py==0.7.4\n",
            "  Downloading ale_py-0.7.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.6 MB 46.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (7.1.2)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.7/dist-packages (from stable-baselines3[extra]) (4.6.0.66)\n",
            "Collecting rich\n",
            "  Downloading rich-12.6.0-py3-none-any.whl (237 kB)\n",
            "\u001b[K     |████████████████████████████████| 237 kB 71.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from ale-py==0.7.4->stable-baselines3[extra]) (5.10.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (7.1.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.28.1)\n",
            "Collecting AutoROM.accept-rom-license\n",
            "  Downloading AutoROM.accept-rom-license-0.4.2.tar.gz (9.8 kB)\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata~=4.13->stable-baselines3[extra]) (4.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata~=4.13->stable-baselines3[extra]) (3.10.0)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (2.14.1)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (1.8.1)\n",
            "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (3.19.6)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (3.4.1)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (57.4.0)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (1.3.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (0.38.3)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (1.50.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (0.6.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (0.4.6)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.9.1->stable-baselines3[extra]) (1.0.1)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (5.2.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (4.9)\n",
            "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (1.15.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (0.2.8)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.9.1->stable-baselines3[extra]) (1.3.1)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.9.1->stable-baselines3[extra]) (0.4.8)\n",
            "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.1.1)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (1.24.3)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2022.9.24)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.9.1->stable-baselines3[extra]) (3.2.2)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (1.4.4)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (2.8.2)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (3.0.9)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3[extra]) (0.11.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->stable-baselines3[extra]) (2022.6)\n",
            "Collecting commonmark<0.10.0,>=0.9.0\n",
            "  Downloading commonmark-0.9.1-py2.py3-none-any.whl (51 kB)\n",
            "\u001b[K     |████████████████████████████████| 51 kB 8.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: pygments<3.0.0,>=2.6.0 in /usr/local/lib/python3.7/dist-packages (from rich->stable-baselines3[extra]) (2.6.1)\n",
            "Building wheels for collected packages: gym, AutoROM.accept-rom-license\n",
            "  Building wheel for gym (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gym: filename=gym-0.21.0-py3-none-any.whl size=1616826 sha256=a68b49d629aa7ca00688f48dd9c9d0169b8d8c6608e5ce204ae4acd72bd17fd3\n",
            "  Stored in directory: /root/.cache/pip/wheels/76/ee/9c/36bfe3e079df99acf5ae57f4e3464ff2771b34447d6d2f2148\n",
            "  Building wheel for AutoROM.accept-rom-license (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for AutoROM.accept-rom-license: filename=AutoROM.accept_rom_license-0.4.2-py3-none-any.whl size=441026 sha256=27f28b5de21f3f23f77cf0b44bffbff04c2de4907cf3409131ba9808e19831ae\n",
            "  Stored in directory: /root/.cache/pip/wheels/87/67/2e/6147e7912fe37f5408b80d07527dab807c1d25f5c403a9538a\n",
            "Successfully built gym AutoROM.accept-rom-license\n",
            "Installing collected packages: gym, commonmark, AutoROM.accept-rom-license, autorom, stable-baselines3, rich, ale-py\n",
            "  Attempting uninstall: gym\n",
            "    Found existing installation: gym 0.25.2\n",
            "    Uninstalling gym-0.25.2:\n",
            "      Successfully uninstalled gym-0.25.2\n",
            "Successfully installed AutoROM.accept-rom-license-0.4.2 ale-py-0.7.4 autorom-0.4.2 commonmark-0.9.1 gym-0.21.0 rich-12.6.0 stable-baselines3-1.6.2\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "gym"
                ]
              }
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Import Frame Stack Wrapper and Gray Scling Wrapper\n",
        "from gym.wrappers import FrameStack, GrayScaleObservation\n",
        "#Import Vectorization Wrapper\n",
        "from stable_baselines3.common.vec_env import VecFrameStack, DummyVecEnv\n",
        "#Import Matplotlib for visualization impact of frame stacking\n",
        "from matplotlib import pyplot as plt"
      ],
      "metadata": {
        "id": "k0iH4eupdpi_"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#1. Create base envoirnment\n",
        "env = gym_super_mario_bros.make(\"SuperMarioBros-v3\")\n",
        "#2. Simplify the controls\n",
        "env = JoypadSpace(env, SIMPLE_MOVEMENT)\n",
        "#3. GrayScale\n",
        "env = GrayScaleObservation(env, keep_dim = True)\n",
        "#4. Wrap inside the dummy envoirnment\n",
        "env = DummyVecEnv([lambda: env])\n",
        "#5. Stack Frames\n",
        "env = VecFrameStack(env, 4, channels_order = 'last')\n"
      ],
      "metadata": {
        "id": "eSHsvaRiidVV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0290e75d-dbac-47e0-9ef2-a9f098dff9bf"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/gym/core.py:318: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  \"Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\"\n",
            "/usr/local/lib/python3.7/dist-packages/gym/wrappers/step_api_compatibility.py:40: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
            "  \"Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\"\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "env.reset()\n",
        "state, reward, done, info = env.step([3])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iT8ICe740szK",
        "outputId": "1eb9e657-f1d0-434c-84f2-119c5749ff81"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/gym/utils/passive_env_checker.py:228: DeprecationWarning: \u001b[33mWARN: Core environment is written in old step API which returns one bool instead of two. It is recommended to rewrite the environment with new step API. \u001b[0m\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.figure(figsize = (20, 16))\n",
        "for id in range(state.shape[3]):\n",
        "  plt.subplot(1, 4, id + 1)\n",
        "  plt.imshow(state[0][:, :, id])\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "id": "C1_jL0UAkuh6",
        "outputId": "c95ddd1b-3879-4161-a78e-b1f46fbe763e"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1440x1152 with 4 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABIgAAAEGCAYAAAAOrq7MAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZAUlEQVR4nO3da6xld30e4Pc3Zy6+jB17cDDGNtgk7gWqhlgOcRWKaEi4+Ms4lYpAanEjJFcCoiCVqk4jNahSJFo1aUvUoDoC2UQJFDUhuBINcSxSVCGInZQYg2Njgx176hsYjO9z+/fDbOODPXvObe9Za53/80hHZ5+19uW31zn71ex31l6rWmsBAAAAoF87hh4AAAAAgGEpiAAAAAA6pyACAAAA6JyCCAAAAKBzCiIAAACAzimIAAAAADq3tIKoqt5aVXdU1V1Vdc2yHgfgRGQRMAayCBgDWQScSLXWFn+nVStJ7kzy80nuT3Jzkne21r6+8AcDmEMWAWMgi4AxkEXAWpa1B9HrktzVWvtma+1gkk8m2b+kxwKYRxYBYyCLgDGQRcAJ7VzS/Z6f5L5VP9+f5KfnXXl37Wmn5PQljQKcLI/nu99urf3o0HOsIougQ1PPopUzTm87X3L20ocCluvgvQcmnUWJPILt4PB3vpsjjz9Z67nusgqiNVXV1UmuTpJTclp+ut401CjAgvxp+x/3Dj3DRski2H6mnkUrLzkrL/u3vzTwRMBW/c27r5lcFiXyCLabB//db637usv6iNmBJBeu+vmC2bIfaK1d21q7rLV22a7sWdIYQOdkETAGG8qilb32ZASWYs0sSuQR9GxZBdHNSS6pqouraneSdyS5YUmPBTCPLALGQBYBYyCLgBNaykfMWmuHq+p9ST6XZCXJx1prX1vGYwHMI4uAMZBFwBjIImAtSzsGUWvts0k+u6z7B1gPWQSMgSwCxkAWASeyrI+YAQAAADARCiIAAACAzimIAAAAADqnIAIAAADonIIIAAAAoHMKIgAAAIDOKYgAAAAAOqcgAgAAAOicgggAAACgczuHHgAAgOXbfcbB3PgPfnvoMV7kqwfPyfv+7J8OPcZk7Dv3+/n0T3xs6DFe5H8+8XfzH7/4lk3ddscpR/L5N/zWgidan5/74nty6Indgzx2z+TR9iCPFmsMeaQgAgDowI4dLa/YuXfoMV7k0SOPDT3CpOxaOTrK3+PLdm7+91iVwZ5T1SAP2z15tD3Io8UaQx75iBkAAABA5xREAAAAAJ1TEAEAAAB0TkEEAAAA0DkFEQAAAEDnnMUMAIDB7Kqj2X3GwbnrDz29K+3wJk/tsiOpHW3u6p2nHFrKWWNay9JOVXzw8EruPvTE3PWv2HlqdtXKUh77W4eeyNE56/7foVdu+n5bywmf07krO7N3xymbvn9YL3m0MfJo+1EQAQAwmNfsPjV3/MOPz13/+lv/cQ7cv29T933qmc/kwrO/N3f9DX/n09lTuzZ13yfy1NGDec0fv2fh95sk3334jPzcje+fu/4TP/vfcvmS3rv87E3vz9x3ZFtw9JmVEz6n91z++fyrfXcv/oHhBeTRxsij7cdHzAAAAAA6pyACAAAA6JyCCAAAAKBzCiIAAACAzimIAAAAADqnIAIAAADonNPcAwB04NmnduWKO64YeowNe/DRMzd922ee2p1vHd6XU27em7PuOpwHfmYlueDp1I5j50beX1dmR7VFjfoDR1st/D7X6wN3/pOcueeZ5dz5Ek4pvR6/f/dP5fOP/O2l3PehZ7wdGoI8kkdbJo+WYvgJAABYunZoR26/++VDj3FStYM7cujg7uQlLY/u3Jmju47m6JO7ktmbsDueOG/gCRfvwP37cmDoIRbse4/szfce2Tv0GCyQPJJHU7Xd80hBBADAtnbopYdy6KVDTwEgjxg3xyACAAAA6JyCCAAAAKBzCiIAAACAzimIAAAAADqnIAIAAADonIIIAAAAoHMKIgAAAIDOKYgAAAAAOqcgAgAAAOicgggAAACgcwoiAAAAgM4piAAAAAA6t3MrN66qe5I8nuRIksOttcuqal+S/57koiT3JHl7a+27WxsT4MTkETAGsohlWzntcP7e+Q9s6rYHj67k9rtfvuCJGCNZxMkgj7afLRVEM/+otfbtVT9fk+Sm1tqHquqa2c//egGPA7AWeQSMgSxiac4564n80SWf29RtHzv6dF579y8teCJGTBaxVPJo+1nGR8z2J7l+dvn6JFcu4TEA1kMeAWMgi4AxkEXACW21IGpJ/qSq/qKqrp4tO7e19tx+Zg8mOfd4N6yqq6vqlqq65VCe3eIYAJvLI1kELNiWs+jIE0+erFmB7Wsh79PkEfRlqx8xe31r7UBVvTTJjVX116tXttZaVbXj3bC1dm2Sa5PkzNp33OsAbMCm8kgWAQu25Szac9EFsgjYqoW8T5NH0Jct7UHUWjsw+/5wkk8neV2Sh6rqvCSZfX94q0MCrEUeAWMgi4AxkEXAZmy6IKqq06vqjOcuJ3lzktuS3JDkqtnVrkryma0OCXAi8ggYA1kEjIEsAjZrKx8xOzfJp6vqufv5/dbaH1fVzUk+VVXvTnJvkrdvfUyAE5JHwBjIImAMZBGwKZsuiFpr30zyE8dZ/p0kb9rKUAAbIY+AMZBFLMrnfv6/5JTjHx4mp1QlOX1T9/sjO07N/37zf567/gN/sz83//XFm7pvxkMWsUjyqC9bPUg1AACwQK/cuTt7atdS7vsVO/fOXXf6zoNLeUxguuRRX7Z6mnsAAAAAJk5BBAAAZEec0RwYB3k0DAURAACQo6mhRwBIIo+GoiACAAAA6JyCCAAAAKBzCiIAAACAzimIAAAAADqnIAIAAJw1CBgNeTQMBREAAOCsQcBoyKNhKIgAAAAAOqcgAgAAAOicgggAAACgcwoiAAAAgM7tHHoAAADgefvvuDI76uSfweee7+w76Y8JjJs86ouCCAAARuSOb5439AgASeRRb3zEDAAAAKBzCiIAAACAzimIAAAAADqnIAIAAADonIIIAAAAoHMKIgAAAIDOKYgAAAAAOqcgAgAAAOicgggAAACgcwoiAAAAgM4piAAAAAA6pyACAAAA6JyCCAAAAKBzCiIAAACAzimIAAAAADqnIAIAAADonIIIAAAAoHMKIgAAAIDOKYgAAAAAOqcgAgAAAOicgggAAACgcwoiAAAAgM6tWRBV1ceq6uGqum3Vsn1VdWNVfWP2/ezZ8qqqD1fVXVV1a1Vduszhgb7II2AMZBEwBrIIWLT17EF0XZK3vmDZNUluaq1dkuSm2c9J8rYkl8y+rk7ykcWMCZBEHgHjcF1kETC86yKLgAVasyBqrX0hyaMvWLw/yfWzy9cnuXLV8o+3Y76U5KyqOm9RwwJ9k0fAGMgiYAxkEbBomz0G0bmttQdmlx9Mcu7s8vlJ7lt1vftnywCWRR4BYyCLgDGQRcCmbfkg1a21lqRt9HZVdXVV3VJVtxzKs1sdA2BTeSSLgEXbahYdeeLJJU0G9GQR79PkEfRlswXRQ8/tkjj7/vBs+YEkF6663gWzZS/SWru2tXZZa+2yXdmzyTEAtpZHsghYkIVl0cre05c+LLBtLfR9mjyCvmy2ILohyVWzy1cl+cyq5e+aHSX/8iSPrdrFEWAZ5BEwBrIIGANZBGzazrWuUFWfSPLGJOdU1f1Jfi3Jh5J8qqreneTeJG+fXf2zSa5IcleSp5L84hJmBjolj4AxkEXAGMgiYNHWLIhaa++cs+pNx7luS/LerQ4FcDzyCBgDWQSMgSwCFm3LB6kGAAAAYNoURAAAAACdUxABAAAAdE5BBAAAANA5BREAAABA5xREAAAAAJ1TEAEAAAB0TkEEAAAA0DkFEQAAAEDnFEQAAAAAnVMQAQAAAHROQQQAAADQOQURAAAAQOcURAAAAACdUxABAAAAdE5BBAAAANA5BREAAABA5xREAAAAAJ1TEAEAAAB0TkEEAAAA0DkFEQAAAEDnFEQAAAAAnVMQAQAAAHROQQQAAADQOQURAAAAQOcURAAAAACdUxABAAAAdE5BBAAAANA5BREAAABA5xREAAAAAJ1TEAEAAAB0TkEEAAAA0DkFEQAAAEDnFEQAAAAAnVMQAQAAAHROQQQAAADQOQURAAAAQOcURAAAAACdW7MgqqqPVdXDVXXbqmUfrKoDVfWV2dcVq9b9SlXdVVV3VNVbljU40BdZBIyFPALGQBYBi7aePYiuS/LW4yz/T621186+PpskVfXqJO9I8prZbX67qlYWNSzQtesii4BxuC7yCBjedZFFwAKtWRC11r6Q5NF13t/+JJ9srT3bWvtWkruSvG4L8wEkkUXAeMgjYAxkEbBoWzkG0fuq6tbZro1nz5adn+S+Vde5f7YMYFlkETAW8ggYA1kEbMpmC6KPJPmxJK9N8kCS39joHVTV1VV1S1XdcijPbnIMoHOyCBiLLeXR6iw68sSTy5gP6MNC/20kj6AvmyqIWmsPtdaOtNaOJvmdPL974oEkF6666gWzZce7j2tba5e11i7blT2bGQPonCwCxmKrebQ6i1b2nr78gYFtadH/NpJH0JdNFURVdd6qH38hyXNHzr8hyTuqak9VXZzkkiR/vrURAY5PFgFjIY+AMZBFwFbsXOsKVfWJJG9Mck5V3Z/k15K8sapem6QluSfJv0iS1trXqupTSb6e5HCS97bWjixndKAnsggYC3kEjIEsAhZtzYKotfbO4yz+6Amu/+tJfn0rQwG8kCwCxkIeAWMgi4BF28pZzAAAAADYBhREAAAAAJ1TEAEAAAB0TkEEAAAA0DkFEQAAAEDnFEQAAAAAnVMQAQAAAHROQQQAAADQOQURAAAAQOcURAAAAACdUxABAAAAdE5BBAAAANA5BREAAABA5xREAAAAAJ1TEAEAAAB0TkEEAAAA0DkFEQAAAEDnFEQAAAAAnVMQAQAAAHRu59ADAABwEhzekdPuHeaffk9ddChZaYM8NjBC8ghGSUEEANCDluw4PPQQAJFHMFI+YgYAAADQOQURAAAAQOcURAAAAACdUxABAAAAdE5BBAAAANA5ZzEDANguDs//v786VCdxkBc89uEdae0Ep5XeefTkDQOcHPIIJkdBBACwTZx2785Rnjr69HtW5q47sjt5+uKDJ3Ea4GSQRzA9PmIGAAAA0DkFEQAAAEDnFEQAAAAAnVMQAQAAAHROQQQAAADQOQURAAAAQOec5h4AYCLq6ZXs+t78/9/bceQkDrMgO44kux/YNXf9oX1H0/ZM8InxQ1510cM5Y9ezQ4+xIU8f3pU7v/WyoccYLXnEVMmj+RREAAATseNgZff3a5DHPjr/PdOadhyav66O5ITP6dCZlezZ/GMzDj9+5iN5+Z7Hhh5jQ757+LTcGQXRPPKIqZJH8ymIAABYU9vKgQkqSVvUJEDv5BEsh2MQAQAAAHROQQQAAADQuTULoqq6sKo+X1Vfr6qvVdUvz5bvq6obq+obs+9nz5ZXVX24qu6qqlur6tJlPwlg+5NFwBjIImAMZBGwDOvZg+hwkn/ZWnt1ksuTvLeqXp3kmiQ3tdYuSXLT7OckeVuSS2ZfVyf5yMKnBnoki4AxkEXAGMgiYOHWLIhaaw+01v5ydvnxJLcnOT/J/iTXz652fZIrZ5f3J/l4O+ZLSc6qqvMWPjnQFVkEjIEsgs1ZKUcFXiRZBJsnj+bb0FnMquqiJD+Z5MtJzm2tPTBb9WCSc2eXz09y36qb3T9b9sCqZamqq3Osvc4pOW2DYwM9k0XAGCwri3adeXb23rF7aXNPzen370gyZ3tU8sTfOnhS5+H4Tjv76bz9x//v0GMs1Nk7n8o//6kvzl3/R/f8/Xzvkb0ncaLjW2QWze5PHs0hj6ZBHm3eug9SXVV7k/xBkve31r6/el1rrWWDJwtsrV3bWrustXbZruzZyE2BjskiYAyWmUUrp56+wEmB7WzRWTS7nTyCTq2rIKqqXTkWPL/XWvvD2eKHntstcfb94dnyA0kuXHXzC2bLALZEFgFjIIuAMZBFwKKt5yxmleSjSW5vrf3mqlU3JLlqdvmqJJ9ZtfxdsyPlX57ksVW7OQJsiiwCxkAWAWMgi4BlWM8xiH4myT9L8tWq+sps2b9J8qEkn6qqdye5N8nbZ+s+m+SKJHcleSrJLy50YqBXsggYA1kEjIEsAhZuzYKotfZ/ktSc1W86zvVbkvducS6AHyKLgDGQRcAYyCJgGdZ9kGoAAAAAtqcNneYeAIDlajuSI6cMPcVxbPhcSM87sqSTRLZKcsj/d47B00/syZ89fMnQY5xU33/81G3/9yeP1k8ejYc8eoENvF4URAAAI9JWkoM/cnToMV5k5Zl5n2ZZ25FTtvBubg0rT3pDNgpP7s59j5w39BQnVSVZGXqIJZNHGyOPRkIe/bCj63+9KIgAAFjTMt9UAWyEPILlUHECAAAAdE5BBAAAANA5BREAAABA5xREAAAAAJ1TEAEAAAB0TkEEAAAA0DkFEQAAAEDnFEQAAAAAnVMQAQAAAHROQQQAAADQOQURAAAAQOcURAAAAACdUxABAAAAdE5BBAAAANA5BREAAABA5xREAAAAAJ1TEAEAAAB0TkEEAAAA0DkFEQAAAEDnFEQAAAAAnVMQAQAAAHROQQQAAADQOQURAAAAQOcURAAAAACdUxABAAAAdE5BBAAAANA5BREAAABA5xREAAAAAJ2r1trQM6SqHknyZJJvDz3LJp0Tsw9lyvNPefbk+PO/srX2o0MMswiyaHBTnt/sw5FF47Md/6amwuzD2XZZlCRV9XiSO4aeY5O249/UVEx59mTa828pi0ZRECVJVd3SWrts6Dk2w+zDmfL8U549mf7880z5eU159mTa85t9OFOff54pP68pz55Me36zD2fq888z5ec15dmTac8/5dmTac+/1dl9xAwAAACgcwoiAAAAgM6NqSC6dugBtsDsw5ny/FOePZn+/PNM+XlNefZk2vObfThTn3+eKT+vKc+eTHt+sw9n6vPPM+XnNeXZk2nPP+XZk2nPv6XZR3MMIgAAAACGMaY9iAAAAAAYwOAFUVW9taruqKq7quqaoedZj6q6p6q+WlVfqapbZsv2VdWNVfWN2fezh54zSarqY1X1cFXdtmrZcWetYz48+13cWlWXDjf53Nk/WFUHZtv+K1V1xap1vzKb/Y6qesswUz+vqi6sqs9X1der6mtV9cuz5aPf/ieYfTLbf6Nk0XJNOYtmM002j2TRtMii5ZJFw5FF0zO1PJpSFiXTziNZNLrZF7ftW2uDfSVZSXJ3klcl2Z3kr5K8esiZ1jn3PUnOecGy/5Dkmtnla5L8+6HnnM3yhiSXJrltrVmTXJHkfyWpJJcn+fIIZ/9gkg8c57qvnv397Ely8ezvamXg+c9Lcuns8hlJ7pzNOfrtf4LZJ7P9N/h8ZdHyZ51sFp1g/km8HmSRLDoJc8uiYeefxOtBFk0ni2bPYXJ5NKUsms0z2TySRaObfWHbfug9iF6X5K7W2jdbaweTfDLJ/oFn2qz9Sa6fXb4+yZUDzvIDrbUvJHn0BYvnzbo/ycfbMV9KclZVnXdyJn2xObPPsz/JJ1trz7bWvpXkrhz7+xpMa+2B1tpfzi4/nuT2JOdnAtv/BLPPM7rtv0GyaMmmnEXJtPNIFsmigciiJZBFsugk2i55NMosSqadR7Jo+2bR0AXR+UnuW/Xz/TnxExyLluRPquovqurq2bJzW2sPzC4/mOTcYUZbl3mzTuX38b7Z7n0fW7Wb6Khnr6qLkvxkki9nYtv/BbMnE9z+6zDV+WXR8Cb1epBFozfV+WXR8Cb1epBFkzDF5zD1LEom9no4jkm9HmTRiw1dEE3V61trlyZ5W5L3VtUbVq9sx/bnmsTp4aY068xHkvxYktcmeSDJbww7ztqqam+SP0jy/tba91evG/v2P87sk9v+25wsGtakXg+yiCWSRcOa1OtBFrFE2yaLkunNm4m9HmTR8Q1dEB1IcuGqny+YLRu11tqB2feHk3w6x3bTeui5Xc1m3x8ebsI1zZt19L+P1tpDrbUjrbWjSX4nz+8iN8rZq2pXjr14f6+19oezxZPY/sebfWrbfwMmOb8sGtaUXg+yaDImOb8sGtaUXg+yaFIm9xy2QRYlE3k9HM+UXg+yaL6hC6Kbk1xSVRdX1e4k70hyw8AznVBVnV5VZzx3Ocmbk9yWY3NfNbvaVUk+M8yE6zJv1huSvGt2pPbLkzy2aje7UXjB5z1/Ice2fXJs9ndU1Z6qujjJJUn+/GTPt1pVVZKPJrm9tfabq1aNfvvPm31K23+DZNEwRv9aOJGpvB5kkSxaJlk0vKm8HmTRpLIomVgebZMsSibwephnKq8HWbTGtm8DHv28PX9U8Dtz7Ijavzr0POuY91U5diTwv0rytedmTvKSJDcl+UaSP02yb+hZZ3N9Isd2MzuUY585fPe8WXPsyOz/dfa7+GqSy0Y4++/OZrt19gd/3qrr/+ps9juSvG0E2/71ObZr4q1JvjL7umIK2/8Es09m+2/iOcui5c472Sw6wfyTeD3IouH//jf4nGXRcueVRcPNLosm9jWlPJpaFs1mm2weyaLRzb6wbV+zGwEAAADQqaE/YgYAAADAwBREAAAAAJ1TEAEAAAB0TkEEAAAA0DkFEQAAAEDnFEQAAAAAnVMQAQAAAHROQQQAAADQuf8PNlMHev0EYT4AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir LOGS\n",
        "LOGS = './LOGS/'"
      ],
      "metadata": {
        "id": "EwZcshLJrm2J"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from stable_baselines3 import PPO\n",
        "model = PPO('CnnPolicy', env, verbose = 1, tensorboard_log = LOGS, learning_rate = 0.001, n_steps = 512)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d9PU2ZBbr-wx",
        "outputId": "6ad291cb-e102-4df8-e887-0a66302f5b53"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using cuda device\n",
            "Wrapping the env in a VecTransposeImage.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.learn(total_timesteps = 100)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HIFUYF8z0gN4",
        "outputId": "e65ad389-186e-499c-d9bc-0ba75c9447f5"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Logging to ./LOGS/PPO_1\n",
            "----------------------------\n",
            "| time/              |     |\n",
            "|    fps             | 33  |\n",
            "|    iterations      | 1   |\n",
            "|    time_elapsed    | 15  |\n",
            "|    total_timesteps | 512 |\n",
            "----------------------------\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<stable_baselines3.ppo.ppo.PPO at 0x7f4edfd66310>"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.save('mario_automated')"
      ],
      "metadata": {
        "id": "xaoEAKXI1v47"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = PPO.load('mario_automated')\n",
        "state = env.reset()\n",
        "while True:\n",
        "  action, _ = model.predict(state)\n",
        "  state, reward, done, info = env.step(action)\n",
        "  # env.render()\n"
      ],
      "metadata": {
        "id": "AUuw-wI73TCi",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 380
        },
        "outputId": "196c2a62-ae7b-4aa0-8102-6b74de146b48"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-17-39f86a99877c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m   \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m   \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minfo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m   \u001b[0;31m# env.render()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/stable_baselines3/common/base_class.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, observation, state, episode_start, deterministic)\u001b[0m\n\u001b[1;32m    630\u001b[0m             \u001b[0;34m(\u001b[0m\u001b[0mused\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrecurrent\u001b[0m \u001b[0mpolicies\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    631\u001b[0m         \"\"\"\n\u001b[0;32m--> 632\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpolicy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobservation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepisode_start\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdeterministic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    633\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    634\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mset_random_seed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseed\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/stable_baselines3/common/policies.py\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, observation, state, episode_start, deterministic)\u001b[0m\n\u001b[1;32m    329\u001b[0m         \u001b[0;31m#     episode_start = [False for _ in range(self.n_envs)]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    330\u001b[0m         \u001b[0;31m# Switch to eval mode (this affects batch norm / dropout)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 331\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_training_mode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    332\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    333\u001b[0m         \u001b[0mobservation\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvectorized_env\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobs_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobservation\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/stable_baselines3/common/policies.py\u001b[0m in \u001b[0;36mset_training_mode\u001b[0;34m(self, mode)\u001b[0m\n\u001b[1;32m    211\u001b[0m         \u001b[0;34m:\u001b[0m\u001b[0mparam\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mtrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mset\u001b[0m \u001b[0mto\u001b[0m \u001b[0mtraining\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mset\u001b[0m \u001b[0mto\u001b[0m \u001b[0mevaluation\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    212\u001b[0m         \"\"\"\n\u001b[0;32m--> 213\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    214\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    215\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mobs_to_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobservation\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTuple\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mth\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, mode)\u001b[0m\n\u001b[1;32m   1835\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1836\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"training mode is expected to be boolean\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1837\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1838\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1839\u001b[0m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__setattr__\u001b[0;34m(self, name, value)\u001b[0m\n\u001b[1;32m   1208\u001b[0m             type(self).__name__, name))\n\u001b[1;32m   1209\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1210\u001b[0;31m     \u001b[0;32mdef\u001b[0m \u001b[0m__setattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'Module'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1211\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mremove_from\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mdicts_or_sets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1212\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0md\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdicts_or_sets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n"
      ],
      "metadata": {
        "id": "iNmEAxp14J_q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "ScbIAEuAwRJU"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}