PyTorchStuff.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PyTorchStuff.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "lUkIMe91UpQ2",
        "COMYlxfie3cr"
      ],
      "authorship_tag": "ABX9TyMehimDkYPuLRz8cm1LlXEt",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rishuatgithub/MLPy/blob/master/PyTorchStuff.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5rJO_tqEULwj",
        "colab_type": "text"
      },
      "source": [
        "## All about Pytorch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yfGJ9Vq_T6Fs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lUkIMe91UpQ2",
        "colab_type": "text"
      },
      "source": [
        "### Tensors\n",
        "\n",
        "Tensors are similar to NumPy’s ndarrays, with the addition being that Tensors can also be used on a GPU to accelerate computing."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VrS_Uu-gUlSU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "9da8388f-a5af-4fe4-9a95-7cc2016dc792"
      },
      "source": [
        "x = torch.empty(5,3) ## empty\n",
        "x"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[6.5694e-36, 0.0000e+00, 3.3631e-44],\n",
              "        [0.0000e+00,        nan, 0.0000e+00],\n",
              "        [1.1578e+27, 1.1362e+30, 7.1547e+22],\n",
              "        [4.5828e+30, 1.2121e+04, 7.1846e+22],\n",
              "        [9.2198e-39, 7.0374e+22, 0.0000e+00]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "achl8IpXU4G2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "ecdd46d8-14b7-4fa1-f612-273ce1848347"
      },
      "source": [
        "x = torch.randn(5,3) ## random initialized\n",
        "x"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.1648,  0.1203,  1.0604],\n",
              "        [ 0.4551,  0.9178,  1.3501],\n",
              "        [ 0.3136, -0.9561,  1.3928],\n",
              "        [ 2.1873, -1.6890,  0.2575],\n",
              "        [-0.6818, -0.4929,  0.5411]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cSpEI1QoU911",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "e551cfc9-b773-4b4c-d57b-61cdbf7209f9"
      },
      "source": [
        "x = torch.zeros(5,3, dtype=torch.long)\n",
        "x"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0, 0, 0],\n",
              "        [0, 0, 0],\n",
              "        [0, 0, 0],\n",
              "        [0, 0, 0],\n",
              "        [0, 0, 0]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IrnBfi6HVPB3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b2a23b58-8bdf-4939-9313-e2154d886a0d"
      },
      "source": [
        "type(x)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Tensor"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cuT3FGM1W9g5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "f2b9acb6-1a86-40c0-d675-bc26a647967d"
      },
      "source": [
        "x = torch.ones(5,2)\n",
        "x"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1., 1.],\n",
              "        [1., 1.],\n",
              "        [1., 1.],\n",
              "        [1., 1.],\n",
              "        [1., 1.]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NrQ3UdcIVSZi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "068df80c-287c-4dd8-84cb-dcd31b1dd0a5"
      },
      "source": [
        "myarr = [[10,20.2],[30,40]] ## sample data\n",
        "\n",
        "x = torch.tensor(myarr)\n",
        "x"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[10.0000, 20.2000],\n",
              "        [30.0000, 40.0000]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iy1MRdssVsmL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 139
        },
        "outputId": "bfd09c11-0745-4745-d95b-6ca00537bfa0"
      },
      "source": [
        "## create a tensor from an existing tensor\n",
        "\n",
        "x = torch.tensor([[1,2],[3,4]], dtype=torch.int16)\n",
        "print(f\"X tensor: {x}\")\n",
        "\n",
        "y = torch.tensor(x, dtype=torch.float16)\n",
        "print(f\"Y tensor: {y}\")"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "X tensor: tensor([[1, 2],\n",
            "        [3, 4]], dtype=torch.int16)\n",
            "Y tensor: tensor([[1., 2.],\n",
            "        [3., 4.]], dtype=torch.float16)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  \n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pGUx6QwWWbni",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e7759e26-d8d0-404d-f6ce-40d8fb226232"
      },
      "source": [
        "## size of tensor\n",
        "\n",
        "x.size()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([2, 2])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qGJWG2vgXWbQ",
        "colab_type": "text"
      },
      "source": [
        "- Operation on Tensors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "80FHn1TNXNII",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "9bbc2de4-73a1-4537-8910-c758ce2ba390"
      },
      "source": [
        "x = torch.randn(5,2)\n",
        "x"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-0.0173, -0.1190],\n",
              "        [ 0.6001,  0.5685],\n",
              "        [-1.3267, -0.1079],\n",
              "        [-1.7419, -0.6463],\n",
              "        [-0.7563, -1.1716]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EIKYKevzXgQn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "cc1de8bf-1d7e-41c3-a93a-1c3c01a1a43f"
      },
      "source": [
        "y = torch.ones(5,2)\n",
        "y"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1., 1.],\n",
              "        [1., 1.],\n",
              "        [1., 1.],\n",
              "        [1., 1.],\n",
              "        [1., 1.]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4ixIo4zdXlW5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "939fef1a-8b45-4141-a503-9b5b341d0939"
      },
      "source": [
        "x + y ## sum of two tensors"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.9827,  0.8810],\n",
              "        [ 1.6001,  1.5685],\n",
              "        [-0.3267,  0.8921],\n",
              "        [-0.7419,  0.3537],\n",
              "        [ 0.2437, -0.1716]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EMnLXVeIXnNL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "df4ce5fa-cdba-4246-d5a1-5d3cf5336f2f"
      },
      "source": [
        "torch.add(x,y) ## alternative: sum of two tensors"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.9827,  0.8810],\n",
              "        [ 1.6001,  1.5685],\n",
              "        [-0.3267,  0.8921],\n",
              "        [-0.7419,  0.3537],\n",
              "        [ 0.2437, -0.1716]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kTPuAVbdXvaL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        },
        "outputId": "3d999801-43e2-4be6-aeae-9c8366d31648"
      },
      "source": [
        "## In Place addition : Any operation that mutates a tensor in-place is post-fixed with an _. For example: x.copy_(y), x.t_(), will change x.\n",
        "\n",
        "y.add_(x)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[ 0.9827,  0.8810],\n",
              "        [ 1.6001,  1.5685],\n",
              "        [-0.3267,  0.8921],\n",
              "        [-0.7419,  0.3537],\n",
              "        [ 0.2437, -0.1716]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hBm3CBmFYB9s",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d3afa1c4-6f6a-4f35-ee98-ac5517f2c8b9"
      },
      "source": [
        "### Standard numpy operations on tensors\n",
        "\n",
        "x[:1]"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-0.0173, -0.1190]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nl41n-osYDRs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "5185e43d-d568-44b2-bf46-4f2b8934d22c"
      },
      "source": [
        "type(x[:1])"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Tensor"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CFQqZbm9YwVe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "c5201570-2a21-471f-c55b-73de94079f59"
      },
      "source": [
        "### Resize the tensor\n",
        "\n",
        "x = torch.randn(4,4)\n",
        "x"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-1.6968,  1.9364, -2.1796, -0.0819],\n",
              "        [ 1.8027,  0.3540,  1.3269,  0.1532],\n",
              "        [-0.4969,  0.4169,  0.5677,  1.0968],\n",
              "        [ 0.0742,  1.5354,  0.9387,  0.0343]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "peYY_vpyY4dw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "8d7be985-f9c4-40b3-b814-dc710701c19f"
      },
      "source": [
        "y = x.view(16)\n",
        "y"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([-1.6968,  1.9364, -2.1796, -0.0819,  1.8027,  0.3540,  1.3269,  0.1532,\n",
              "        -0.4969,  0.4169,  0.5677,  1.0968,  0.0742,  1.5354,  0.9387,  0.0343])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W5UYOR71ZCFE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "1be02e8a-a614-493d-cb1d-2e81e76c3037"
      },
      "source": [
        "z = x.view([-1,8])\n",
        "z"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-1.6968,  1.9364, -2.1796, -0.0819,  1.8027,  0.3540,  1.3269,  0.1532],\n",
              "        [-0.4969,  0.4169,  0.5677,  1.0968,  0.0742,  1.5354,  0.9387,  0.0343]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v_JjU3pXZLrP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "4c5ca3c7-960d-401f-b50d-fbf10ed37b0c"
      },
      "source": [
        "## transpose an array\n",
        "\n",
        "torch.transpose(x, 0,1)"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[-1.6968,  1.8027, -0.4969,  0.0742],\n",
              "        [ 1.9364,  0.3540,  0.4169,  1.5354],\n",
              "        [-2.1796,  1.3269,  0.5677,  0.9387],\n",
              "        [-0.0819,  0.1532,  1.0968,  0.0343]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D7Av3nCeZsMd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "63f3d811-b8e1-4c9f-8863-084c96a87899"
      },
      "source": [
        "## to get the value of a tensor\n",
        "\n",
        "x = torch.randn(1)\n",
        "print(x)\n",
        "print(x.item())"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([0.7582])\n",
            "0.7581729292869568\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ON-RTJ_TbAXV",
        "colab_type": "text"
      },
      "source": [
        "Numpy Bridge with Tensors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ToJuv3-0axoC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b9091059-141f-4ee9-c2aa-84f03bbaa5f6"
      },
      "source": [
        "a = torch.ones(5)\n",
        "a"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([1., 1., 1., 1., 1.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G2NXBOEbZpv9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "10f93ed2-a522-4474-9c75-0e986e6a3a77"
      },
      "source": [
        "type(a)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Tensor"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kTHZTKYAZSO-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "55e7fe57-9877-43b8-b709-b8e78f1e9498"
      },
      "source": [
        "### convert to numpy\n",
        "\n",
        "a.numpy()"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1., 1., 1., 1., 1.], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wwqMtD1dbeFo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "1cb248ea-e529-44ec-ff5c-ae2fd18cae1e"
      },
      "source": [
        "## converting numpy array to torch tensors\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "a = np.ones(5)\n",
        "t = torch.tensor(a, dtype=torch.int)\n",
        "\n",
        "print(a, type(a))\n",
        "print(t, type(t))"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1. 1. 1. 1. 1.] <class 'numpy.ndarray'>\n",
            "tensor([1, 1, 1, 1, 1], dtype=torch.int32) <class 'torch.Tensor'>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A8-b3FkJcNb5",
        "colab_type": "text"
      },
      "source": [
        "- Running on the Device"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3vAeH5mIbwHZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "2df1c3af-5f9a-4c2f-808d-1651a640a03c"
      },
      "source": [
        "### if CUDA is available or not\n",
        "\n",
        "torch.cuda.is_available()"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9HDfRFypcZam",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "536de6f6-8590-4254-b445-204c6bee9973"
      },
      "source": [
        "if torch.cuda.is_available():\n",
        "  device = torch.device(\"cuda\") ## define device\n",
        "  \n",
        "  x = torch.ones(5) ## normal stuff\n",
        "  print(x)\n",
        "  \n",
        "  y = torch.ones_like(x, device=device) ### running it on gpu\n",
        "  print(y)\n",
        "\n",
        "  x = x.to(device)  ## change the execution to device\n",
        "\n",
        "  z = x + y\n",
        "\n",
        "  print(z)\n",
        "  \n",
        "  print(z.to(\"cpu\", dtype=torch.int32)) ## change the data type of z using .to and run it on cpu"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([1., 1., 1., 1., 1.])\n",
            "tensor([1., 1., 1., 1., 1.], device='cuda:0')\n",
            "tensor([2., 2., 2., 2., 2.], device='cuda:0')\n",
            "tensor([2, 2, 2, 2, 2], dtype=torch.int32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "COMYlxfie3cr",
        "colab_type": "text"
      },
      "source": [
        "### AUTOGRAD"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Xfd6EDSdMCM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "03849fc7-60d2-4610-fd4d-256d9b8df19d"
      },
      "source": [
        "x = torch.ones(2,2, requires_grad=True)\n",
        "x"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[1., 1.],\n",
              "        [1., 1.]], requires_grad=True)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HvzKRjyegGtW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "239636b5-2ffc-4649-c9e1-b69007a2703f"
      },
      "source": [
        "y = x + 2\n",
        "y"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[3., 3.],\n",
              "        [3., 3.]], grad_fn=<AddBackward0>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mUvAtKbLgMpd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "5299885a-a9b4-45f7-f63f-f8133e11de3d"
      },
      "source": [
        "y.grad_fn ### y was created as a result of an operation, hence it has a grad_fn"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<AddBackward0 at 0x7f99e6c6dba8>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H4BEqWfRgUr6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "71432c1d-3fed-4ddd-81f4-0409fde899b0"
      },
      "source": [
        "## more operation on y\n",
        "\n",
        "z = y*y*3\n",
        "out = z.mean()\n",
        "\n",
        "print(z, out)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[27., 27.],\n",
            "        [27., 27.]], grad_fn=<MulBackward0>) tensor(27., grad_fn=<MeanBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rudrfwL-gpvY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "5fab86f0-21d5-42c3-c665-07bcb627edfc"
      },
      "source": [
        "### .requires_grad_( ... ) changes an existing Tensor’s requires_grad flag in-place. The input flag defaults to False if not given.\n",
        "\n",
        "a = torch.randn(2,2)\n",
        "a = ((a*2)/(a-1))\n",
        "print(a.requires_grad)\n",
        "\n",
        "a.requires_grad_(True) ## changing the grad inplace\n",
        "\n",
        "print(a.requires_grad)\n",
        "\n",
        "b = (a * a).sum()\n",
        "print(b.grad_fn)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "False\n",
            "True\n",
            "<SumBackward0 object at 0x7f99e6c77908>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VPXed2bTindH",
        "colab_type": "text"
      },
      "source": [
        "- Gradient"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XL0LZ4PZiDHg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "9b8ed208-5684-4bda-8b86-e3f9d73f5283"
      },
      "source": [
        "out ## out contains a single scalar, out.backward() is equivalent to out.backward(torch.tensor(1.))."
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(27., grad_fn=<MeanBackward0>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8jnD77E4kZ7J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "out.backward()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i02nWgjEkiWg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "5f45bf69-d796-4e1c-a2c2-093a2fbae36f"
      },
      "source": [
        "print(x.grad) ### Print gradients d(out)/dx"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[4.5000, 4.5000],\n",
            "        [4.5000, 4.5000]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zkkmeSmvklB0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "6fcecda2-c9b5-4259-ff58-3377c1a2e159"
      },
      "source": [
        "## another example\n",
        "\n",
        "t1 = torch.ones(1, requires_grad= True)\n",
        "t2 = torch.ones(1, requires_grad=True)\n",
        "\n",
        "print(t1, t2)\n",
        "\n",
        "s = t1+t2\n",
        "print(s)"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([1.], requires_grad=True) tensor([1.], requires_grad=True)\n",
            "tensor([2.], grad_fn=<AddBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_wAw3BNmrZTm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "961829ee-ed05-4ee5-e7aa-c6c409426691"
      },
      "source": [
        "s.grad_fn"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<AddBackward0 at 0x7f99e6c77080>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_rj4H5Glr89U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "s.backward()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UvGpO3zusBEm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7ee8abcf-60a1-4bf9-f76b-671e858a873d"
      },
      "source": [
        "t1.grad"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([1.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lCAhC15bsrbq",
        "colab_type": "text"
      },
      "source": [
        "Vector Jacobian Product"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TKlVsvcYsvny",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "97797fa3-e2ea-4c11-e5bd-6eae8b49d714"
      },
      "source": [
        "x = torch.randn(3, requires_grad=True)\n",
        "\n",
        "y = x * 2\n",
        "while y.data.norm() < 1000:\n",
        "    y = y * 2\n",
        "\n",
        "print(y)"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([-674.6850, -785.7428, -353.3467], grad_fn=<MulBackward0>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AE8FrXBCsNxj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "05789a2a-2fef-48ec-e075-a58595f1f12d"
      },
      "source": [
        "v = torch.tensor([0.1, 1.0, 0.0001], dtype=torch.float)\n",
        "y.backward(v)\n",
        "\n",
        "print(x.grad)"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([2.0480e+02, 2.0480e+03, 2.0480e-01])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MjKp1LzzvEW5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "f7727c8c-ef34-494e-bcc1-8416af4a7fa4"
      },
      "source": [
        "print(x.requires_grad)\n",
        "y = x.detach()\n",
        "print(y.requires_grad)\n",
        "print(x.eq(y).all())"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "True\n",
            "False\n",
            "tensor(True)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n0nRgVYzL3Dt",
        "colab_type": "text"
      },
      "source": [
        "### Neural Network"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OC_-rLutMSH4",
        "colab_type": "text"
      },
      "source": [
        "A typical training procedure for a neural network is as follows:\n",
        "\n",
        "1.   Define the neural network that has some learnable parameters (or weights)\n",
        "1.   Iterate over a dataset of inputs\n",
        "2.   Process input through the network\n",
        "3.   Compute the loss (how far is the output from being correct)\n",
        "4.   Propagate gradients back into the network’s parameters\n",
        "5.   Update the weights of the network, typically using a simple update rule:\n",
        "```\n",
        "weight = weight - learning_rate * gradient\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HCE4fl5fNBvd",
        "colab_type": "text"
      },
      "source": [
        "- Simple NN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4EpzmnKXLtJH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.nn as nn\n",
        "import torch.nn.functional as F"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j5mqjkmDNP_U",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "c55f67f4-b30b-4483-b6ac-fcd595e12f48"
      },
      "source": [
        "X = torch.tensor(([2, 9], [5, 1], [3, 6]), dtype=torch.float) ## 3x2 tensor\n",
        "y = torch.tensor(([92], [100], [89]), dtype=torch.float)  ## 3x1 tensor\n",
        "xPredicted = torch.tensor(([4, 8]), dtype=torch.float) # 1 X 2 tensor\n",
        "\n",
        "print(X)\n",
        "print(y)"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[2., 9.],\n",
            "        [5., 1.],\n",
            "        [3., 6.]])\n",
            "tensor([[ 92.],\n",
            "        [100.],\n",
            "        [ 89.]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TW05wNdFTHwn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "5cfbf1c5-85fb-41a4-8246-8d764bbedbad"
      },
      "source": [
        "X_max, X_max_ind = torch.max(X, 0) ## return max including indices and max values per col. 0 for col, 1 for row\n",
        "print(X_max, X_max_ind)\n",
        "\n",
        "xPredicted_max, _ = torch.max(xPredicted, 0)\n",
        "print(xPredicted_max)\n",
        "\n",
        "y_max = torch.max(y)\n",
        "print(y_max)\n",
        "\n",
        "## scaling\n",
        "X = torch.div(X, X_max)\n",
        "xPredicted = torch.div(xPredicted, xPredicted_max)\n",
        "y = y/y_max\n",
        "\n",
        "print(f\"X is : {X}\")\n",
        "print(f\"xPredicted is : {xPredicted}\")\n",
        "print(f\"y is : {y}\")"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([1., 1.]) tensor([1, 0])\n",
            "tensor(1.)\n",
            "tensor(1.)\n",
            "X is : tensor([[0.4000, 1.0000],\n",
            "        [1.0000, 0.1111],\n",
            "        [0.6000, 0.6667]])\n",
            "xPredicted is : tensor([0.5000, 1.0000])\n",
            "y is : tensor([[0.9200],\n",
            "        [1.0000],\n",
            "        [0.8900]])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z0WyDiUbVn-u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class SimpleNN(nn.Module):\n",
        "  def __init__(self):\n",
        "    super(SimpleNN, self).__init__()\n",
        "    ## parameters\n",
        "    self.input_size = 2\n",
        "    self.hidden_layer = 3\n",
        "    self.output_layer = 1\n",
        "\n",
        "    ## initializing the weights\n",
        "    self.W1 = torch.randn(self.input_size, self.hidden_layer) # 2x3 tensor\n",
        "    self.W2 = torch.randn(self.hidden_layer, self.output_layer) # 3x1 tensor\n",
        "\n",
        "\n",
        "  def forward(self, X):\n",
        "    '''\n",
        "      Forward propagation\n",
        "    '''\n",
        "    self.z = torch.matmul(X, self.W1)\n",
        "    self.z2 = torch.sigmoid(self.z)\n",
        "    \n",
        "    self.z3 = torch.matmul(self.z2, self.W2)\n",
        "    o = torch.sigmoid(self.z3) ## final activation function\n",
        "\n",
        "    return o\n",
        "  \n",
        "\n",
        "  def sigmoid(self, s):\n",
        "    return 1 / (1 + torch.exp(-s))\n",
        "    \n",
        "  \n",
        "  def sigmoidPrime(self, s):\n",
        "    # derivative of sigmoid\n",
        "    return s * (1 - s)\n",
        "\n",
        "  \n",
        "  def backward(self, X, y, o):\n",
        "    '''\n",
        "      Backward propagation\n",
        "    '''\n",
        "    self.o_error = y - o ## calculate the difference b/w predicted and actual\n",
        "    \n",
        "    self.o_delta = self.o_error * self.sigmoidPrime(o) ## derivative of sig to error\n",
        "    self.z2_error = torch.matmul(self.o_delta, torch.t(self.W2))\n",
        "\n",
        "    self.z2_delta = self.z2_error * self.sigmoidPrime(self.z2)\n",
        "\n",
        "    self.W1 += torch.matmul(torch.t(X), self.z2_delta)\n",
        "    self.W2 += torch.matmul(torch.t(self.z2), self.o_delta)\n",
        "\n",
        "  \n",
        "  def train(self, X, y):\n",
        "    # forward + backward pass for training\n",
        "    o = self.forward(X)\n",
        "    self.backward(X, y, o)\n",
        "\n",
        "  def saveWeights(self, model):\n",
        "    # we will use the PyTorch internal storage functions\n",
        "    torch.save(model, \"NN\")\n",
        "        \n",
        "  def predict(self):\n",
        "    print (\"Predicted data based on trained weights: \")\n",
        "    print (\"Input (scaled):\" + str(xPredicted))\n",
        "    print (\"Output:\" + str(self.forward(xPredicted)))\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c1R0aa7vUDsP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "b5ea9c75-b20d-4b68-c87a-7e52af5faae3"
      },
      "source": [
        "NN = SimpleNN()\n",
        "\n",
        "print(NN)"
      ],
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SimpleNN()\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ffNqGIljcPod",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 292
        },
        "outputId": "090f9962-34c5-416e-b8e4-83a2a8c8cf2f"
      },
      "source": [
        "for i in range(10):  # trains the NN 10 times\n",
        "    print (\"#\" + str(i) + \" Loss: \" + str(torch.mean((y - NN(X))**2).detach().item()))  # mean sum squared loss\n",
        "    NN.train(X, y)\n",
        "NN.saveWeights(NN)\n",
        "NN.predict()"
      ],
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "#0 Loss: 0.2509533762931824\n",
            "#1 Loss: 0.178019180893898\n",
            "#2 Loss: 0.12616543471813202\n",
            "#3 Loss: 0.09139183163642883\n",
            "#4 Loss: 0.068324513733387\n",
            "#5 Loss: 0.05276729539036751\n",
            "#6 Loss: 0.041979026049375534\n",
            "#7 Loss: 0.03426816314458847\n",
            "#8 Loss: 0.028596043586730957\n",
            "#9 Loss: 0.02431400679051876\n",
            "Predicted data based on trained weights: \n",
            "Input (scaled):tensor([0.5000, 1.0000])\n",
            "Output:tensor([0.8484])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/torch/serialization.py:402: UserWarning: Couldn't retrieve source code for container of type SimpleNN. It won't be checked for correctness upon loading.\n",
            "  \"type \" + obj.__name__ + \". It won't be checked \"\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1U0TplHvdQsp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}