00_pytorch_fundamentals.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyNE/h4mKDF0UNDHjtrq/aca",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ErikLarssonDev/PyTorch/blob/main/00_pytorch_fundamentals.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 00. Pytorch Fundamentals"
      ],
      "metadata": {
        "id": "vnbqP-U8hMlM"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "print(torch.__version__)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rLrU1osvg6Nw",
        "outputId": "8781b5a5-1675-41e0-eaa6-71aaaf6f0de5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.0.1+cu118\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Introduction to Tensors\n",
        "\n",
        "### Creating tensors\n",
        "\n",
        "PyTorch tensors are created using ```torch.tensor()``` https://pytorch.org/docs/stable/tensors.html\n"
      ],
      "metadata": {
        "id": "3TpR0zdUizFR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# scalar\n",
        "scalar = torch.tensor(7)\n",
        "scalar"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-Hp1gKqbjN6G",
        "outputId": "add9b0b2-7c9d-4176-e499-32d49ca7d320"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(7)"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "scalar.ndim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PdvdxVkGjZpC",
        "outputId": "4bd6b36a-f091-4eb1-a6df-7fca360d1d36"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get tensor back as python int\n",
        "scalar.item()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GdDUMZU1j8Nz",
        "outputId": "f6048305-9818-4caa-a278-ee93c095e285"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "7"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Vector\n",
        "vector = torch.tensor([7,7])\n",
        "vector"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2A6etePdkAgL",
        "outputId": "ce70ffa3-d920-41bd-8b8c-ac9ebda6ab5e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([7, 7])"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vector.ndim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1oUfF2ZjkJgP",
        "outputId": "bd02ff7d-288a-4bac-b140-a4bfcd91bb50"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vector.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HZ8XYMTYkXDf",
        "outputId": "0d81e3c2-a938-48d0-b0ba-6b904074ad78"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2])"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# MATRIX\n",
        "MATRIX = torch.tensor([[7,8],\n",
        "                       [9,10]])\n",
        "MATRIX"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ni2lTh7tkb5m",
        "outputId": "a76861a8-965b-4ade-893c-8494f2f85e3c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 7,  8],\n",
              "        [ 9, 10]])"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "MATRIX.ndim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vZThiBpZksY4",
        "outputId": "09ef6ab4-470b-4b65-e50d-05e16e17420b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "MATRIX[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s-qU6gXhkzfU",
        "outputId": "7d2b8961-d086-4410-f774-cb334a4fd549"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([7, 8])"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "MATRIX[1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cfP72CVMk1Yw",
        "outputId": "cf067c3d-11ca-4299-b229-0033b3a6f8db"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 9, 10])"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "MATRIX.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PVIrkNBLk3yL",
        "outputId": "413f2365-ea94-4177-e4af-4e1a70373d27"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 2])"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# TENSOR\n",
        "TENSOR = torch.tensor([[[1,2,3],\n",
        "                        [3,6,9],\n",
        "                        [2,4,5]]])\n",
        "TENSOR"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I6wOCvvuk7JD",
        "outputId": "ddbabaff-9ccd-4c23-b7f6-5dead30bbe8b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[[1, 2, 3],\n",
              "         [3, 6, 9],\n",
              "         [2, 4, 5]]])"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "TENSOR.ndim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VxQCS-9ClO-_",
        "outputId": "03215439-0092-49fa-f872-f42333502843"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "TENSOR.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "U2bgdhZalXKK",
        "outputId": "d282a9d3-f73d-47b0-8fee-eaa6629c5f8d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 3, 3])"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "TENSOR[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GRI8YdISlZGL",
        "outputId": "e66431a8-bcd7-4294-d0e2-f3d86f8497b4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1, 2, 3],\n",
              "        [3, 6, 9],\n",
              "        [2, 4, 5]])"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Random tensors\n",
        "\n",
        "Why random tensors?\n",
        "\n",
        "Random tensors are important because the way many neural networks learn is that they start with tensors full of random numbers and then adjust those random numbers to better represent the data.\n",
        "\n",
        "`Start with random numbers -> look at data -> update random numbers -> look at data -> update random numbers.`\n"
      ],
      "metadata": {
        "id": "4aNIcBNcliqY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a random tensor of size (3,4)\n",
        "random_tensor = torch.rand(3,4)\n",
        "random_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TSwGREQPmguK",
        "outputId": "af343bfb-253f-4227-f727-0fbf52b52682"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.9879, 0.9819, 0.5601, 0.5894],\n",
              "        [0.2590, 0.8048, 0.0923, 0.1063],\n",
              "        [0.9361, 0.7588, 0.3600, 0.8597]])"
            ]
          },
          "metadata": {},
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "random_tensor.ndim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1msrWwRHnQOf",
        "outputId": "9c97487c-3bab-4f25-9391-8dcd78133e3c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "3"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a random tensor with similar shape to an image tensor\n",
        "random_image_size_tensor = torch.rand(size=(224, 244, 3)) # height, width, color channels (RGB)\n",
        "random_image_size_tensor.shape, random_image_size_tensor.ndim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xS6Hc8WQnU6l",
        "outputId": "e86cc651-e92b-4ccc-917f-7405b56d3fb6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([224, 244, 3]), 3)"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Zeros and ones"
      ],
      "metadata": {
        "id": "7KIh6Qnun7GS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensor of all zeros\n",
        "zeros = torch.zeros(3,4)\n",
        "zeros"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "59uCLE-NoyA-",
        "outputId": "c76512f0-4c19-43c3-d713-2e393fc71cd7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0., 0., 0., 0.],\n",
              "        [0., 0., 0., 0.],\n",
              "        [0., 0., 0., 0.]])"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "zeros * random_tensor # A zeros tensor will create a zero tensor by multiplication with another tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OGz2nS2vo3km",
        "outputId": "36d6b1c6-8263-4173-be2f-48f6a8efc607"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0., 0., 0., 0.],\n",
              "        [0., 0., 0., 0.],\n",
              "        [0., 0., 0., 0.]])"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensor of all ones\n",
        "ones = torch.ones(3,4)\n",
        "ones"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zfjq4BoppBNw",
        "outputId": "81d54876-6551-4d4e-b1f8-cc114e0a38fd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1., 1., 1., 1.],\n",
              "        [1., 1., 1., 1.],\n",
              "        [1., 1., 1., 1.]])"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "ones.dtype"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Yz5xYuR_pQLQ",
        "outputId": "f040ccaf-ca9a-4362-e990-51c6a75de376"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.float32"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "random_tensor.dtype # torch.float32 is standard data type"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4kPooTeWpSbE",
        "outputId": "21cba8f2-542f-47d4-8167-76d174c7731e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.float32"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Creating a range of tensors and tensors-like"
      ],
      "metadata": {
        "id": "F_Y6J6DhpWAH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Use torch.range()\n",
        "torch.range(0,10) # deprecated"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MF_SqGM1pnlH",
        "outputId": "6ffa296c-6fd2-4fb8-fbfc-f436e62ba239"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-38-a70e3231c961>:2: UserWarning: torch.range is deprecated and will be removed in a future release because its behavior is inconsistent with Python's range builtin. Instead, use torch.arange, which produces values in [start, end).\n",
            "  torch.range(0,10)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 0.,  1.,  2.,  3.,  4.,  5.,  6.,  7.,  8.,  9., 10.])"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "one_to_ten = torch.arange(1,11)\n",
        "one_to_ten"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r1LpOjubpq93",
        "outputId": "f790ac79-1358-4847-e003-ccda96aba321"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10])"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Creating tensors like\n",
        "ten_zeros = torch.zeros_like(one_to_ten)\n",
        "ten_zeros"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tlkNWzrcpyh_",
        "outputId": "4cdf1a35-7667-446a-9e43-71fecf078c9e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
            ]
          },
          "metadata": {},
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Tensor datatypes\n",
        "\n",
        "**Note:** Tensor datatypes is one of the three big errors you'll run into with PyTorch & deep learning:\n",
        "1. Tensors not right datatype\n",
        "2. Tensors not right shape\n",
        "3. Tensors not on the right device"
      ],
      "metadata": {
        "id": "_aoxixrWqV_I"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Float 32 tensor\n",
        "float_32_tensor = torch.tensor([3.0, 6.0, 9.0],\n",
        "                               dtype=None, # What datatype, default 32-bit float\n",
        "                               device=None, # What device is your tensor on, default cpu\n",
        "                               requires_grad=False) # If you want PyTorch to track the gradient\n",
        "float_32_tensor, float_32_tensor.dtype"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C6rfJb2bqy_L",
        "outputId": "ead9ab8b-e77c-42b7-b589-2f6c8d3978ce"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([3., 6., 9.]), torch.float32)"
            ]
          },
          "metadata": {},
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "float_16_tensor = float_32_tensor.type(torch.float16)\n",
        "float_16_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pArf1B8pq_y_",
        "outputId": "c6098c66-145a-4a71-ba2e-739f7a8b0375"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([3., 6., 9.], dtype=torch.float16)"
            ]
          },
          "metadata": {},
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "float_16_tensor * float_32_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ogd9L6fMst6N",
        "outputId": "acfd006f-111a-48db-ec4e-d46c6265fcb2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 9., 36., 81.])"
            ]
          },
          "metadata": {},
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "int_32_tensor = torch.tensor([3, 6, 9], dtype=torch.int32)\n",
        "int_32_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XBdiSggPs9ZC",
        "outputId": "0d261397-345e-4fae-ec2c-8e8ac25000b4"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([3, 6, 9], dtype=torch.int32)"
            ]
          },
          "metadata": {},
          "execution_count": 51
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "float_32_tensor * int_32_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uFYIWOVctXSh",
        "outputId": "d187a84f-68e5-480f-99b1-466f3b3087df"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([ 9., 36., 81.])"
            ]
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Getting information from tensors (tensor attributes)\n",
        "\n",
        "1. Tensors not right datatype - to get datatype from a tensor, can use `tensor.dtype`\n",
        "2. Tensors not right shape - to get shape from a tensor, can use `tensor.shape`\n",
        "3. Tensors not on the right device - to get the device from a tensor, can use `tensor.device`\n",
        "\n"
      ],
      "metadata": {
        "id": "sUohZAwttbg0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensor\n",
        "some_tensor = torch.rand(3,4)\n",
        "some_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lqMV-MnmuJeb",
        "outputId": "57a375eb-cdf7-4cd6-c4d4-73f6fbed1f66"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.2992, 0.1109, 0.2474, 0.0224],\n",
              "        [0.6140, 0.7256, 0.1881, 0.6163],\n",
              "        [0.8516, 0.0717, 0.3353, 0.2854]])"
            ]
          },
          "metadata": {},
          "execution_count": 53
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find out details about some tensor\n",
        "print(some_tensor)\n",
        "print(f\"Datatype of tensor: {some_tensor.dtype}\")\n",
        "print(f\"Shape of tensor: {some_tensor.shape}\")\n",
        "print(f\"Device tensor is on: {some_tensor.device}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XyJIVzmFuQW3",
        "outputId": "749ad90b-ebd8-45fb-9cf0-4493fb22889d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[0.2992, 0.1109, 0.2474, 0.0224],\n",
            "        [0.6140, 0.7256, 0.1881, 0.6163],\n",
            "        [0.8516, 0.0717, 0.3353, 0.2854]])\n",
            "Datatype of tensor: torch.float32\n",
            "Shape of tensor: torch.Size([3, 4])\n",
            "Device tensor is on: cpu\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Manipulating Tensors (tensor operations)\n",
        "\n",
        "Tensor operations inclide:\n",
        "* Addition\n",
        "* Subtraction\n",
        "* Multiplication (element-wise)\n",
        "* Division\n",
        "* Matrix multiplication"
      ],
      "metadata": {
        "id": "qqKzDT7EufSn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensor and add 10 to it\n",
        "tensor = torch.tensor([1, 2, 3])\n",
        "tensor + 10"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K35Q73k2vgMI",
        "outputId": "ca2c9a1f-9e47-4188-8816-8e124288a72e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([11, 12, 13])"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Multiply tensor by 10\n",
        "tensor * 10"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H2Dfr01tvxGe",
        "outputId": "22f61ffb-908d-41e9-9672-456616facbb0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([10, 20, 30])"
            ]
          },
          "metadata": {},
          "execution_count": 59
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Subtract 10\n",
        "tensor - 10"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9ixSmq6xv4jR",
        "outputId": "528a71f7-0dab-47fe-8f2b-f92b1f8694af"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([-9, -8, -7])"
            ]
          },
          "metadata": {},
          "execution_count": 60
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Try out PyTorch in-built functions\n",
        "torch.mul(tensor, 10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EQ3cWAH7wGH2",
        "outputId": "1ccbcb2c-7da9-4627-8cd0-9f10568ffa5a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([10, 20, 30])"
            ]
          },
          "metadata": {},
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.add(tensor, 10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "orOGsKQ-wNdG",
        "outputId": "3fbb7439-296b-4d67-893e-7a2fc3d5cb86"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([11, 12, 13])"
            ]
          },
          "metadata": {},
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Matrix multiplication\n",
        "\n",
        "Two main ways of performing multiplication in neural networks and deep learning:\n",
        "1. Element-wise multiplication\n",
        "2. Matrix multiplication (dot product)\n",
        "\n",
        "There are two main rules that performing matrix multiplication needs to satisfy:\n",
        "1. The **inner dimensions** must match.\n",
        "* `(3,2) @ (3,2)` won't work\n",
        "* `(2,3) @ (3,2)` will work\n",
        "* `(3,2) @ (2,3)` will work\n",
        "2. The resulting matrix has the shape of the **outer dimensions**:\n",
        "* `(2,3) @ (3,2)` -> `(2,2)`\n",
        "* `(3,3) @ (3,3)` -> `(3,3)`"
      ],
      "metadata": {
        "id": "G6Os94HUwUr5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Element-wise multiplication\n",
        "print(tensor, \"*\", tensor)\n",
        "print(f\"Equals: {tensor * tensor}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-YWAtu-5xbEX",
        "outputId": "fdf7fb3a-4ed3-4294-cba5-14ed91f4704e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([1, 2, 3]) * tensor([1, 2, 3])\n",
            "Equals: tensor([1, 4, 9])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Matrix multiplication\n",
        "torch.matmul(tensor, tensor)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vUu0ITaoyEOI",
        "outputId": "27fb75eb-dea6-49f0-d585-f528a6b53ff2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(14)"
            ]
          },
          "metadata": {},
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "value = 0\n",
        "for i in range(len(tensor)):\n",
        "  value += tensor[i] * tensor[i]\n",
        "print(value)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eo6MWa79yJFY",
        "outputId": "10683a72-8667-4158-fcf2-5cf93f97e3e2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(14)\n",
            "CPU times: user 1.95 ms, sys: 3 µs, total: 1.95 ms\n",
            "Wall time: 1.91 ms\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%time\n",
        "torch.matmul(tensor, tensor)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nr2QRjDsycvR",
        "outputId": "f52ba721-44bb-4973-f023-f28681c069fb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 485 µs, sys: 0 ns, total: 485 µs\n",
            "Wall time: 393 µs\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(14)"
            ]
          },
          "metadata": {},
          "execution_count": 68
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### One of the most common errors in deep learning: shape errors"
      ],
      "metadata": {
        "id": "rzkYeAVfy8Ap"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Shapes for matrix multiplication\n",
        "tensor_A = torch.tensor([[1, 2],\n",
        "                         [3, 4],\n",
        "                         [5, 6]])\n",
        "tensor_B = torch.tensor([[7, 10],\n",
        "                         [8, 11],\n",
        "                         [9, 12]])\n",
        "# torch.mm is the same as torch.matmul\n",
        "torch.mm(tensor_A, tensor_B.T) # Need to transpose to ensure compatible shapes"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MTnn-wUf1csi",
        "outputId": "ef600603-0a52-43b4-bbe8-e8de243bd281"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 27,  30,  33],\n",
              "        [ 61,  68,  75],\n",
              "        [ 95, 106, 117]])"
            ]
          },
          "metadata": {},
          "execution_count": 74
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Finding the min, max, mean, sum, etc (tensor aggregation)"
      ],
      "metadata": {
        "id": "9BegNdvj3H33"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensor\n",
        "x = torch.arange(0, 100, 10)\n",
        "x, x.dtype"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PAr-HtnvBGCX",
        "outputId": "187304a0-2d0a-4d33-adaf-14b17fec310c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([ 0, 10, 20, 30, 40, 50, 60, 70, 80, 90]), torch.int64)"
            ]
          },
          "metadata": {},
          "execution_count": 80
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the min\n",
        "torch.min(x), x.min()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I5x0rMOCBQuU",
        "outputId": "2c7d7df2-138f-4d1a-db94-101042b02638"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(0), tensor(0))"
            ]
          },
          "metadata": {},
          "execution_count": 77
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the max\n",
        "torch.max(x), x.max()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YzcsTuXRBaU8",
        "outputId": "f4dd0e63-3591-48b8-e424-a514d881f8cd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(90), tensor(90))"
            ]
          },
          "metadata": {},
          "execution_count": 78
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the mean\n",
        "torch.mean(x.type(torch.float32)), x.type(torch.float32).mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FW7rM5WXBf8L",
        "outputId": "a8999008-4ec1-4102-d197-8ca0a6626b99"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(45.), tensor(45.))"
            ]
          },
          "metadata": {},
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the sum\n",
        "torch.sum(x), x.sum()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iMFuYZF1Biiw",
        "outputId": "ddb814e2-2274-41ae-e473-3e7677ecbf55"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(450), tensor(450))"
            ]
          },
          "metadata": {},
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Finding the positional min and max"
      ],
      "metadata": {
        "id": "rwL0nhreCVoV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the position in tensor with the minimum value\n",
        "x.argmin()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I9BBHUrZChkK",
        "outputId": "5b66acf0-5526-4345-cc83-a5771eb6c0ae"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(0)"
            ]
          },
          "metadata": {},
          "execution_count": 88
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Find the position in tensor with the maximum value\n",
        "x.argmax()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jYaNV3OoCyYp",
        "outputId": "10c49f61-a60a-4081-ee95-5be2a405931c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(9)"
            ]
          },
          "metadata": {},
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Reshaping, stacking, squeezing and unsqueezing tensors\n",
        "\n",
        "* Reshaping - reshapes and input tensor to a defined shape\n",
        "* View - return a view of an input tensor of certain shape but keep the same memory as the original tensor\n",
        "* Stacking - combine multiple tensors on top of each other (vstack) or beside each other (hstack)\n",
        "* Squeeze - remove all `1` dimensions from a tensor\n",
        "* Unsqueeze - add a `1` dimension to a target tensor\n",
        "* Permute - Return a view of the input with dimensions permuted (swapped) in a certain way"
      ],
      "metadata": {
        "id": "F3drtQS4Czjq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's create a tensor\n",
        "x = torch.arange(1., 10.)\n",
        "x, x.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C1GYJmg6EdSM",
        "outputId": "988d7e7d-f68a-45f6-af17-bae5bd1e6087"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([1., 2., 3., 4., 5., 6., 7., 8., 9.]), torch.Size([9]))"
            ]
          },
          "metadata": {},
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Add an extra dimension\n",
        "x_reshaped = x.reshape(1, 9)\n",
        "x_reshaped, x_reshaped.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AFgQjgm4Eid6",
        "outputId": "a9956713-6471-4988-ec75-ee3817379dda"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[1., 2., 3., 4., 5., 6., 7., 8., 9.]]), torch.Size([1, 9]))"
            ]
          },
          "metadata": {},
          "execution_count": 97
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Change the view\n",
        "z = x.view(1, 9)\n",
        "z, z.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uJzPaCg5EuyU",
        "outputId": "c6107b6a-c168-4d7d-b21c-2163028ec67b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[1., 2., 3., 4., 5., 6., 7., 8., 9.]]), torch.Size([1, 9]))"
            ]
          },
          "metadata": {},
          "execution_count": 98
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Shares the same memory as x, changing z changes x\n",
        "z[:, 0] = 5\n",
        "z, x"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "glCy8hoQFikm",
        "outputId": "10ed861b-d865-47c1-8ca4-55649e940629"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[5., 2., 3., 4., 5., 6., 7., 8., 9.]]),\n",
              " tensor([5., 2., 3., 4., 5., 6., 7., 8., 9.]))"
            ]
          },
          "metadata": {},
          "execution_count": 99
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Stack tensors on top of each other\n",
        "x_stacked = torch.stack([x, x, x, x], dim=0)\n",
        "x_stacked"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dAexsh0kFwi6",
        "outputId": "fbc13b47-1ee9-4af2-93f3-9ba29f3793dc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[5., 2., 3., 4., 5., 6., 7., 8., 9.],\n",
              "        [5., 2., 3., 4., 5., 6., 7., 8., 9.],\n",
              "        [5., 2., 3., 4., 5., 6., 7., 8., 9.],\n",
              "        [5., 2., 3., 4., 5., 6., 7., 8., 9.]])"
            ]
          },
          "metadata": {},
          "execution_count": 104
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Stack tensors side by side\n",
        "x_stacked = torch.stack([x, x, x, x], dim=1)\n",
        "x_stacked"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vta4GJ-4GGrj",
        "outputId": "8abafc85-3f37-440f-c35c-0f6a0f4f5cef"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[5., 5., 5., 5.],\n",
              "        [2., 2., 2., 2.],\n",
              "        [3., 3., 3., 3.],\n",
              "        [4., 4., 4., 4.],\n",
              "        [5., 5., 5., 5.],\n",
              "        [6., 6., 6., 6.],\n",
              "        [7., 7., 7., 7.],\n",
              "        [8., 8., 8., 8.],\n",
              "        [9., 9., 9., 9.]])"
            ]
          },
          "metadata": {},
          "execution_count": 105
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# torch.squeeze() - removes all single dimensions from a target tensor\n",
        "print(f\"Previous tensor: {x_reshaped}\")\n",
        "print(f\"Previous shape: {x_reshaped.shape}\")\n",
        "\n",
        "x_squeezed = x_reshaped.squeeze()\n",
        "print(f\"\\nNew tensor: {x_squeezed}\")\n",
        "print(f\"New shape: {x_squeezed.shape}\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yFDuTHkYGTcu",
        "outputId": "9a1b8e29-504e-41b9-8153-6a22d1e3ff53"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Previous tensor: tensor([[5., 2., 3., 4., 5., 6., 7., 8., 9.]])\n",
            "Previous shape: torch.Size([1, 9])\n",
            "\n",
            "New tensor: tensor([5., 2., 3., 4., 5., 6., 7., 8., 9.])\n",
            "New shape: torch.Size([9])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# torch.unsqueeze() - adds a single dimension to a target tensor at a specific dim (dimension)\n",
        "print(f\"Previous tensor: {x_squeezed}\")\n",
        "print(f\"Previous shape: {x_squeezed.shape}\")\n",
        "\n",
        "x_unsqueezed = x_squeezed.unsqueeze(dim=0)\n",
        "print(f\"\\nNew tensor: {x_unsqueezed}\")\n",
        "print(f\"New shape: {x_unsqueezed.shape}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EqAFnzFnGyU2",
        "outputId": "12640aab-4339-431c-ed23-8e6b562fc159"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Previous tensor: tensor([5., 2., 3., 4., 5., 6., 7., 8., 9.])\n",
            "Previous shape: torch.Size([9])\n",
            "\n",
            "New tensor: tensor([[5., 2., 3., 4., 5., 6., 7., 8., 9.]])\n",
            "New shape: torch.Size([1, 9])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# torch.permute - rearranges the dimensions of a target tensor in a specified order\n",
        "x_original = torch.rand(size=(224, 224, 3)) # [height, width, color_channels]\n",
        "\n",
        "# Permute the original tensor to rearrange the axis (or dim) order\n",
        "x_permuted = x_original.permute(2, 0, 1) # Shifts axis 0 ->, 1->2, 2->0\n",
        "print(f\"Previous shape: {x_original.shape}\")\n",
        "print(f\"New shape: {x_permuted.shape}\") # Shares the same reference in memory as original"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JgVNMgImHtmP",
        "outputId": "3a03ceef-d382-47d5-ccf1-8a7dfb81104d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Previous shape: torch.Size([224, 224, 3])\n",
            "New shape: torch.Size([3, 224, 224])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Indexing (selecting data from tensors)\n",
        "\n",
        "Indexing with PyTorch is similar to indexing with NumPy\n",
        "\n"
      ],
      "metadata": {
        "id": "nY_-DUI3Iv9Y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensor\n",
        "x = torch.arange(1, 10).reshape(1, 3, 3)\n",
        "x, x.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QBG0Cd1_J44n",
        "outputId": "a779095d-d883-4804-8985-8df7e764aba5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([[[1, 2, 3],\n",
              "          [4, 5, 6],\n",
              "          [7, 8, 9]]]),\n",
              " torch.Size([1, 3, 3]))"
            ]
          },
          "metadata": {},
          "execution_count": 115
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's index on our new tensor\n",
        "x[0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M0-MyYThKFys",
        "outputId": "fb2f2e67-3554-4b0a-e1da-a04297d28281"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1, 2, 3],\n",
              "        [4, 5, 6],\n",
              "        [7, 8, 9]])"
            ]
          },
          "metadata": {},
          "execution_count": 116
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's index on the middle bracket (dim=1)\n",
        "x[0][0], x[0, 0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VdQfdcIhKOUD",
        "outputId": "092af1c0-4ed5-4be4-ba17-9b81074d977b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([1, 2, 3]), tensor([1, 2, 3]))"
            ]
          },
          "metadata": {},
          "execution_count": 117
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's index on the most inner bracket (last dimension)\n",
        "x[0][0][0], x[0, 0, 0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "stvWkKxiKZvY",
        "outputId": "e18bbcd5-e95f-4252-fb7f-caf32f168c03"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(1), tensor(1))"
            ]
          },
          "metadata": {},
          "execution_count": 118
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# You can alsoe use \":\" to select \"all\" of a target dimension\n",
        "x[:, 0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hiOhv74VKjR1",
        "outputId": "35eb65fd-3607-46bd-a8df-362fb124a993"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1, 2, 3]])"
            ]
          },
          "metadata": {},
          "execution_count": 119
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get all values of 0th of 1st dimensions but only index 1 of 2nd dimension\n",
        "x[:, :, 1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hSJ2-TLTLBG1",
        "outputId": "14798945-410f-4b48-db5e-04625cbd1a74"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[2, 5, 8]])"
            ]
          },
          "metadata": {},
          "execution_count": 120
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get all values of the 0 dimension but only the 1 index value of the 1st and 2nd dimension\n",
        "x[:, 1, 1]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sm25hUOOLJQm",
        "outputId": "8593578a-c051-42f0-9c78-215728f4a56e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([5])"
            ]
          },
          "metadata": {},
          "execution_count": 121
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get index 0 of 0th and 1st dimension and all values of 2nd dimension\n",
        "x[0, 0, :]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xaDsklDxLV3g",
        "outputId": "60405494-9e98-47a0-fbf7-ca5d6650e494"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([1, 2, 3])"
            ]
          },
          "metadata": {},
          "execution_count": 122
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Index on x to return 9\n",
        "print(x[0, 2, 2])\n",
        "\n",
        "# Index on x to return 3, 6, 9\n",
        "print(x[0, :, 2])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "42f4KINVLkJZ",
        "outputId": "8eb0de6a-7d17-40a6-d71d-cff03a5770cf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(9)\n",
            "tensor([3, 6, 9])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## PyTorch tensors & NumPy\n",
        "\n",
        "NumPy is a popular scientific Python numerical computing library and because of this, PyTorch has functionality to interact with it.\n",
        "\n",
        "* Data in NumPy, want in PyTorch tensor -> `torch.from_numpy(ndarray)`\n",
        "* PyTorch tensor -> NumPy -> `torch.Tensor.numpy()`\n",
        "\n"
      ],
      "metadata": {
        "id": "JmUU6iSfLtyY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Numpy array to tensor\n",
        "array = np.arange(1.0, 8.0) # Default float64\n",
        "tensor = torch.from_numpy(array)\n",
        "array, tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q55oUhUcMvik",
        "outputId": "1f988a7d-17ed-4dce-b32d-dc4fe6ea408c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([1., 2., 3., 4., 5., 6., 7.]),\n",
              " tensor([1., 2., 3., 4., 5., 6., 7.], dtype=torch.float64))"
            ]
          },
          "metadata": {},
          "execution_count": 125
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Change the value of array, what will this do to `tensor`?\n",
        "array = array + 1\n",
        "array, tensor # Does not affect tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pX1sXbydM8PR",
        "outputId": "bad25502-c559-4a14-97d7-20bb580a47f8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([3., 4., 5., 6., 7., 8., 9.]),\n",
              " tensor([1., 2., 3., 4., 5., 6., 7.], dtype=torch.float64))"
            ]
          },
          "metadata": {},
          "execution_count": 127
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Tensor to NumPy array\n",
        "tensor = torch.ones(7)\n",
        "numpy_tensor = tensor.numpy()\n",
        "tensor, numpy_tensor # Will get the dtype of float32 as it is default for tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4yMoSm1eNlKP",
        "outputId": "9a8a220d-974a-4a57-e2df-8531b1084b53"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([1., 1., 1., 1., 1., 1., 1.]),\n",
              " array([1., 1., 1., 1., 1., 1., 1.], dtype=float32))"
            ]
          },
          "metadata": {},
          "execution_count": 129
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Change the tensor, what happens to `numpy_tensor``\n",
        "tensor = tensor + 1\n",
        "tensor, numpy_tensor # Does not change numpy_tensor"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a46BUXBjN0_r",
        "outputId": "357dc37f-7036-4c92-ac33-21baa04f46c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor([2., 2., 2., 2., 2., 2., 2.]),\n",
              " array([1., 1., 1., 1., 1., 1., 1.], dtype=float32))"
            ]
          },
          "metadata": {},
          "execution_count": 130
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Reproducibility (trying to take the random out of random)\n",
        "\n",
        "In short how a neural network learns\n",
        "`start with random numbers -> tensor operations -> update random numbers to try and make them better representations of the data -> again -> again...`\n",
        "\n",
        "To reduce the randomness in neural networks and PyTorch comes the concept of a **random seed**.\n",
        "\n",
        "Essentially what the random seed does is \"flavour\" the randomness.\n",
        "\n"
      ],
      "metadata": {
        "id": "VWLZpcOZOINS"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create two random tensors\n",
        "random_tensor_A = torch.rand(3,4)\n",
        "random_tensor_B = torch.rand(3,4)\n",
        "\n",
        "print(random_tensor_A)\n",
        "print(random_tensor_B)\n",
        "print(random_tensor_A == random_tensor_B)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7AKxQYE_O5G7",
        "outputId": "cc850657-c57f-4dc5-94b7-d40b9cbd8a2a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[0.3560, 0.2705, 0.6708, 0.5879],\n",
            "        [0.8853, 0.5471, 0.9001, 0.2900],\n",
            "        [0.7903, 0.6217, 0.5782, 0.4498]])\n",
            "tensor([[0.6823, 0.8507, 0.9198, 0.3199],\n",
            "        [0.1675, 0.8519, 0.0175, 0.3424],\n",
            "        [0.9845, 0.6670, 0.7644, 0.1409]])\n",
            "tensor([[False, False, False, False],\n",
            "        [False, False, False, False],\n",
            "        [False, False, False, False]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's make some random but reproducable tensors\n",
        "\n",
        "# Set the random seed\n",
        "RANDOM_SEED = 42\n",
        "torch.manual_seed(RANDOM_SEED)\n",
        "random_tensor_C = torch.rand(3,4)\n",
        "\n",
        "torch.manual_seed(RANDOM_SEED) # Need to set this before each random method\n",
        "random_tensor_D = torch.rand(3,4)\n",
        "\n",
        "print(random_tensor_C)\n",
        "print(random_tensor_D)\n",
        "print(random_tensor_C == random_tensor_D)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_X6q42tfPPhA",
        "outputId": "6c0e357e-97c9-46a3-8382-8a5e7056220f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[0.8823, 0.9150, 0.3829, 0.9593],\n",
            "        [0.3904, 0.6009, 0.2566, 0.7936],\n",
            "        [0.9408, 0.1332, 0.9346, 0.5936]])\n",
            "tensor([[0.8823, 0.9150, 0.3829, 0.9593],\n",
            "        [0.3904, 0.6009, 0.2566, 0.7936],\n",
            "        [0.9408, 0.1332, 0.9346, 0.5936]])\n",
            "tensor([[True, True, True, True],\n",
            "        [True, True, True, True],\n",
            "        [True, True, True, True]])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Running tensors and PyTorch objects on the GPUs (and making faster computations)\n",
        "\n",
        "GPUs = faster computation on numbers, thanks to CUDA + NVIDIA hardware + PyTorch working behind the scenes."
      ],
      "metadata": {
        "id": "wspOWjrQPvvH"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 1. Getting a GPU\n",
        "1. Easiest - Use Google Colab for a free GPU (options to upgrade as well)\n",
        "2. Use your own GPU - takes a little bit of setup and requires the investment of putchasing a GPU\n",
        "3. Use cloud computing - GCP, AWS, Azure, these services allow you to rent computers and access them\n"
      ],
      "metadata": {
        "id": "9ArYiIFzRJEA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!nvidia-smi"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wwmCXghtSyvD",
        "outputId": "d2575311-db5b-44eb-bbc9-a392633d81c2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mon Jul 17 14:42:03 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.105.17   Driver Version: 525.105.17   CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   34C    P8    11W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 2. Check for GPU access with PyTorch"
      ],
      "metadata": {
        "id": "6oNObIVeTRyg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Check for the GPU access with PyTorch\n",
        "torch.cuda.is_available()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Uegqe7CaTiNS",
        "outputId": "0bbc0df8-053f-44c3-a946-ffadea6886b2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 136
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Setup device agnostic code\n",
        "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
        "device"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "1oZyum5BTue4",
        "outputId": "4de445db-b1b1-4024-c7e7-f6545e6dc101"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'cuda'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 139
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Count number of devices\n",
        "torch.cuda.device_count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oNFe8KWhUBaM",
        "outputId": "05f1b02f-ccad-4e4b-d219-11d7abb55cf3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {},
          "execution_count": 140
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 3. Putting tensors (and models) on the GPU\n",
        "\n",
        "The reason we want our tensors/models on the GPU is because using a GPU results in faster computations.\n",
        "\n"
      ],
      "metadata": {
        "id": "AWZLrX8DUNh1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create a tensor (default on the CPU)\n",
        "tensor = torch.tensor([1, 2, 3])\n",
        "\n",
        "print(tensor, tensor.device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BlpE_sDeU5UI",
        "outputId": "4319d80f-bef7-47fb-960a-f41271cacad7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([1, 2, 3], device='cuda:0') cuda:0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Move tensor to GPU (if available)\n",
        "tensor_on_gpu = tensor.to(device)\n",
        "tensor_on_gpu"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oBb5q6kVVGUz",
        "outputId": "b6be5304-d242-47ef-9806-56f0e3d337d7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([1, 2, 3], device='cuda:0')"
            ]
          },
          "metadata": {},
          "execution_count": 143
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### 4. Moving tensors back to the CPU"
      ],
      "metadata": {
        "id": "tEyhZOpqVWx3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# If tensor is on GPU, can't transform it to NumPy\n",
        "tensor_on_cpu = tensor_on_gpu.cpu().numpy() # To fix we must first set it to the CPU\n",
        "tensor_on_cpu"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hWLVAtAqVo5T",
        "outputId": "f6f0f8ad-17c9-484a-a40e-27fda1342257"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 2, 3])"
            ]
          },
          "metadata": {},
          "execution_count": 147
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "U6cNKOequ6GQ"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}