character_level_GPT/v1/GPT_character_level_v1_trained.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a39a114c",
   "metadata": {
    "papermill": {
     "duration": 0.010865,
     "end_time": "2023-02-13T01:38:30.100002",
     "exception": false,
     "start_time": "2023-02-13T01:38:30.089137",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# **Shakespeare GPT**\n",
    "---\n",
    "\n",
    "from Andrej Karpathy's Tutorial: [ Let's build GPT: from scratch, in code, spelled out. ](https://www.youtube.com/watch?v=kCc8FmEb1nY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a9850816",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:30.120545Z",
     "iopub.status.busy": "2023-02-13T01:38:30.120043Z",
     "iopub.status.idle": "2023-02-13T01:38:31.677601Z",
     "shell.execute_reply": "2023-02-13T01:38:31.676569Z"
    },
    "papermill": {
     "duration": 1.570752,
     "end_time": "2023-02-13T01:38:31.680225",
     "exception": false,
     "start_time": "2023-02-13T01:38:30.109473",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fead77e7e10>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from dataclasses import dataclass\n",
    "\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "from pathlib import Path\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "torch.manual_seed(1357)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "291d356e",
   "metadata": {
    "papermill": {
     "duration": 0.00624,
     "end_time": "2023-02-13T01:38:31.693232",
     "exception": false,
     "start_time": "2023-02-13T01:38:31.686992",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cfc00d3d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:31.709339Z",
     "iopub.status.busy": "2023-02-13T01:38:31.707537Z",
     "iopub.status.idle": "2023-02-13T01:38:31.726986Z",
     "shell.execute_reply": "2023-02-13T01:38:31.726116Z"
    },
    "papermill": {
     "duration": 0.029232,
     "end_time": "2023-02-13T01:38:31.729037",
     "exception": false,
     "start_time": "2023-02-13T01:38:31.699805",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('/kaggle/input/shakespeare-full-text/shakespeare.txt','r',encoding='utf-8') as f:\n",
    "    data = f.read()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0812c1e4",
   "metadata": {
    "papermill": {
     "duration": 0.00623,
     "end_time": "2023-02-13T01:38:31.742025",
     "exception": false,
     "start_time": "2023-02-13T01:38:31.735795",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Character Level Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "604d64b7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:31.758773Z",
     "iopub.status.busy": "2023-02-13T01:38:31.757269Z",
     "iopub.status.idle": "2023-02-13T01:38:31.777896Z",
     "shell.execute_reply": "2023-02-13T01:38:31.777018Z"
    },
    "papermill": {
     "duration": 0.029909,
     "end_time": "2023-02-13T01:38:31.779893",
     "exception": false,
     "start_time": "2023-02-13T01:38:31.749984",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class CharacterLevelTokenizer:\n",
    "    def __init__(self,data):\n",
    "        self.data = data\n",
    "        self.vocab = sorted(list(set(self.data)))\n",
    "        self.VOCAB_SIZE = len(self.vocab)\n",
    "        \n",
    "        self.i_s = {i:s for i,s in enumerate(self.vocab)}\n",
    "        self.s_i = {s:i for i,s in self.i_s.items()}\n",
    "        \n",
    "    def encode(self,s):\n",
    "        return torch.tensor([self.s_i[c] for c in s],dtype=torch.long)\n",
    "\n",
    "    def decode(self,s):\n",
    "        return ''.join([self.i_s[i.item()] for i in s])\n",
    "\n",
    "tokenizer = CharacterLevelTokenizer(data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5a5990e",
   "metadata": {
    "papermill": {
     "duration": 0.006054,
     "end_time": "2023-02-13T01:38:31.792245",
     "exception": false,
     "start_time": "2023-02-13T01:38:31.786191",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ce3836c6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:31.806623Z",
     "iopub.status.busy": "2023-02-13T01:38:31.806334Z",
     "iopub.status.idle": "2023-02-13T01:38:31.872976Z",
     "shell.execute_reply": "2023-02-13T01:38:31.872009Z"
    },
    "papermill": {
     "duration": 0.0762,
     "end_time": "2023-02-13T01:38:31.875175",
     "exception": false,
     "start_time": "2023-02-13T01:38:31.798975",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cuda'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@dataclass\n",
    "class Config:\n",
    "    block_size = 256 # context-length\n",
    "    batch_size = 64 # mini-batch size\n",
    "    vocab_size = tokenizer.VOCAB_SIZE\n",
    "    n_embed = 128\n",
    "    n_heads = 4\n",
    "    head_size = n_embed // n_heads # computes to 384/6=64 or 128/4=32\n",
    "    \n",
    "    n_layers = 3\n",
    "    \n",
    "    train_iters = 10_000\n",
    "    val_iters = 1000\n",
    "    lr = 3e-4\n",
    "    \n",
    "    attn_dropout = 0.1\n",
    "    block_dropout = 0.2\n",
    "    \n",
    "    device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "    \n",
    "Config.device"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0a8f467",
   "metadata": {
    "papermill": {
     "duration": 0.006355,
     "end_time": "2023-02-13T01:38:31.888490",
     "exception": false,
     "start_time": "2023-02-13T01:38:31.882135",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "db4a796e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:31.902632Z",
     "iopub.status.busy": "2023-02-13T01:38:31.902319Z",
     "iopub.status.idle": "2023-02-13T01:38:31.911082Z",
     "shell.execute_reply": "2023-02-13T01:38:31.910026Z"
    },
    "papermill": {
     "duration": 0.018504,
     "end_time": "2023-02-13T01:38:31.913380",
     "exception": false,
     "start_time": "2023-02-13T01:38:31.894876",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ShakespeareDataset:\n",
    "    def __init__(self,Config, is_test=False) -> None:\n",
    "        self.tokenizer = CharacterLevelTokenizer(data)\n",
    "        self.is_test = is_test\n",
    "        self.full_data = self.tokenizer.encode(self.tokenizer.data)\n",
    "        if self.is_test:\n",
    "            self.data = self.full_data[int(0.9*len(self.full_data)):]\n",
    "        else:\n",
    "            self.data = self.full_data[:int(0.9*len(self.full_data))]\n",
    "        self.block_size = Config.block_size\n",
    "        self.batch_size = Config.batch_size\n",
    "\n",
    "    def __len__(self) -> int:\n",
    "        return len(self.data)\n",
    "\n",
    "    def get_block_size(self) -> int:\n",
    "        return self.block_size\n",
    "\n",
    "    def get_vocab_size(self) -> int:\n",
    "        return self.tokenizer.VOCAB_SIZE\n",
    "\n",
    "    def get(self):\n",
    "        ix = torch.randint(len(self.data) - self.block_size, (self.batch_size,))\n",
    "        x = torch.stack([self.data[i:i+self.block_size] for i in ix])\n",
    "        y = torch.stack([self.data[i+1:i+self.block_size+1] for i in ix])\n",
    "        return x,y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67d35adc",
   "metadata": {
    "papermill": {
     "duration": 0.006509,
     "end_time": "2023-02-13T01:38:31.926266",
     "exception": false,
     "start_time": "2023-02-13T01:38:31.919757",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "---\n",
    "\n",
    "# **GPT Model**"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "173472b9",
   "metadata": {
    "papermill": {
     "duration": 0.010409,
     "end_time": "2023-02-13T01:38:31.949683",
     "exception": false,
     "start_time": "2023-02-13T01:38:31.939274",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Single Attention Head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4f4a3adf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:31.976167Z",
     "iopub.status.busy": "2023-02-13T01:38:31.975782Z",
     "iopub.status.idle": "2023-02-13T01:38:31.991392Z",
     "shell.execute_reply": "2023-02-13T01:38:31.989316Z"
    },
    "papermill": {
     "duration": 0.030997,
     "end_time": "2023-02-13T01:38:31.994680",
     "exception": false,
     "start_time": "2023-02-13T01:38:31.963683",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class AttentionHead(nn.Module):\n",
    "    def __init__(self, Config):\n",
    "        super().__init__()\n",
    "\n",
    "        self.block_size = Config.block_size\n",
    "        self.n_embed = Config.n_embed\n",
    "        self.head_size = Config.head_size\n",
    "        \n",
    "        self.key = nn.Linear(self.n_embed, self.head_size, bias=False)\n",
    "        self.query = nn.Linear(self.n_embed, self.head_size, bias=False)\n",
    "        \n",
    "        self.value = nn.Linear(self.n_embed, self.head_size, bias=False)\n",
    "\n",
    "        self.register_buffer(\n",
    "            'tril',\n",
    "            torch.tril(torch.ones(self.block_size,self.block_size))\n",
    "        )\n",
    "        \n",
    "        self.dropout = nn.Dropout(Config.attn_dropout)\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        B,T,C = x.shape\n",
    "\n",
    "        k = self.key(x)\n",
    "        q = self.query(x)\n",
    "\n",
    "        wei = q@k.transpose(-2,-1) * (C ** 0.5)\n",
    "        wei = wei.masked_fill(self.tril[:T,:T]==0,float('-inf'))\n",
    "        wei = F.softmax(wei, dim=-1)\n",
    "        wei = self.dropout(wei)\n",
    "        \n",
    "        v = self.value(x)\n",
    "        out = wei @ v\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e47e5592",
   "metadata": {
    "papermill": {
     "duration": 0.006153,
     "end_time": "2023-02-13T01:38:32.008510",
     "exception": false,
     "start_time": "2023-02-13T01:38:32.002357",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## MultiHead Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "da1d4e2c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:32.022502Z",
     "iopub.status.busy": "2023-02-13T01:38:32.022196Z",
     "iopub.status.idle": "2023-02-13T01:38:32.028831Z",
     "shell.execute_reply": "2023-02-13T01:38:32.027899Z"
    },
    "papermill": {
     "duration": 0.015969,
     "end_time": "2023-02-13T01:38:32.030821",
     "exception": false,
     "start_time": "2023-02-13T01:38:32.014852",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MultiHeadAttention(nn.Module):\n",
    "    def __init__(self, Config):\n",
    "        super().__init__()\n",
    "        self.n_heads = Config.n_heads\n",
    "        self.head_size = Config.head_size\n",
    "        \n",
    "        self.heads = nn.ModuleList([AttentionHead(Config) for _ in range(self.n_heads)])\n",
    "        \n",
    "        self.projection = nn.Linear(Config.n_embed, Config.n_embed)\n",
    "        \n",
    "        self.dropout = nn.Dropout(Config.attn_dropout)\n",
    "    \n",
    "    def forward(self,x):\n",
    "        x = torch.cat([h(x) for h in self.heads],dim=-1)\n",
    "        x = self.projection(x)\n",
    "        x = self.dropout(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe4fbcdb",
   "metadata": {
    "papermill": {
     "duration": 0.006092,
     "end_time": "2023-02-13T01:38:32.043188",
     "exception": false,
     "start_time": "2023-02-13T01:38:32.037096",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Feed Forward Layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7faf110a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:32.057041Z",
     "iopub.status.busy": "2023-02-13T01:38:32.056771Z",
     "iopub.status.idle": "2023-02-13T01:38:32.062481Z",
     "shell.execute_reply": "2023-02-13T01:38:32.061495Z"
    },
    "papermill": {
     "duration": 0.01487,
     "end_time": "2023-02-13T01:38:32.064425",
     "exception": false,
     "start_time": "2023-02-13T01:38:32.049555",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class FeedForward(nn.Module):\n",
    "    def __init__(self, Config):\n",
    "        super().__init__()\n",
    "        self.net = nn.Sequential(\n",
    "            nn.Linear(Config.n_embed,Config.n_embed * 4),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(Config.n_embed * 4, Config.n_embed), # projection\n",
    "            nn.Dropout(Config.block_dropout)\n",
    "        )\n",
    "    def forward(self,x):\n",
    "        return self.net(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a7f2114",
   "metadata": {
    "papermill": {
     "duration": 0.006202,
     "end_time": "2023-02-13T01:38:32.076904",
     "exception": false,
     "start_time": "2023-02-13T01:38:32.070702",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Transformer Block"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "eb4f9397",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:32.091298Z",
     "iopub.status.busy": "2023-02-13T01:38:32.090373Z",
     "iopub.status.idle": "2023-02-13T01:38:32.096684Z",
     "shell.execute_reply": "2023-02-13T01:38:32.095863Z"
    },
    "papermill": {
     "duration": 0.015397,
     "end_time": "2023-02-13T01:38:32.098592",
     "exception": false,
     "start_time": "2023-02-13T01:38:32.083195",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class TransformerBlock(nn.Module):\n",
    "    def __init__(self, Config):\n",
    "        super().__init__()\n",
    "        self.attn = MultiHeadAttention(Config)\n",
    "        self.ff = FeedForward(Config)\n",
    "        self.ln1 = nn.LayerNorm(Config.n_embed)\n",
    "        self.ln2 = nn.LayerNorm(Config.n_embed)\n",
    "\n",
    "    def forward(self,x):\n",
    "        x = x + self.attn(self.ln1(x))\n",
    "        x = x + self.ff(self.ln2(x))\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84cdae0c",
   "metadata": {
    "papermill": {
     "duration": 0.006187,
     "end_time": "2023-02-13T01:38:32.111516",
     "exception": false,
     "start_time": "2023-02-13T01:38:32.105329",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## GPT Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "554b6d26",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:32.126712Z",
     "iopub.status.busy": "2023-02-13T01:38:32.125251Z",
     "iopub.status.idle": "2023-02-13T01:38:32.135102Z",
     "shell.execute_reply": "2023-02-13T01:38:32.134261Z"
    },
    "papermill": {
     "duration": 0.019262,
     "end_time": "2023-02-13T01:38:32.137084",
     "exception": false,
     "start_time": "2023-02-13T01:38:32.117822",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ShakespeareGPT(nn.Module):\n",
    "    def __init__(self,Config):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.n_embed = Config.n_embed\n",
    "        self.block_size = Config.block_size\n",
    "        \n",
    "        self.token_embedding_table = nn.Embedding(Config.vocab_size,self.n_embed)\n",
    "        self.pos_embedding_table = nn.Embedding(self.block_size, self.n_embed)\n",
    "        \n",
    "        self.blocks = nn.Sequential(\n",
    "            *[TransformerBlock(Config)]*Config.n_layers,\n",
    "            nn.LayerNorm(self.n_embed)\n",
    "        )\n",
    "\n",
    "        self.lm_head = nn.Linear(self.n_embed,Config.vocab_size)\n",
    "        \n",
    "    def forward(self,idx):\n",
    "        \n",
    "        B,T = idx.shape\n",
    "        \n",
    "        token_embs = self.token_embedding_table(idx)\n",
    "        pos_embs = self.pos_embedding_table(torch.arange(T,device=Config.device))\n",
    "        \n",
    "        \n",
    "        x = token_embs + pos_embs\n",
    "        x = self.blocks(x)\n",
    "        logits = self.lm_head(x)\n",
    "        \n",
    "        return logits\n",
    "\n",
    "        \n",
    "    def generate(self,idx,total):\n",
    "        for _ in range(total):\n",
    "            idx_cond = idx[:, -self.block_size:]\n",
    "            logits= self(idx_cond)\n",
    "            logits = logits[:, -1, :]\n",
    "            probs = F.softmax(logits, dim=-1)\n",
    "            idx_next = torch.multinomial(probs, num_samples=1)\n",
    "            idx = torch.cat((idx, idx_next), dim=1)\n",
    "        return idx\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6109482",
   "metadata": {
    "papermill": {
     "duration": 0.006111,
     "end_time": "2023-02-13T01:38:32.149496",
     "exception": false,
     "start_time": "2023-02-13T01:38:32.143385",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c20826a5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:32.163432Z",
     "iopub.status.busy": "2023-02-13T01:38:32.163168Z",
     "iopub.status.idle": "2023-02-13T01:38:32.447592Z",
     "shell.execute_reply": "2023-02-13T01:38:32.446588Z"
    },
    "papermill": {
     "duration": 0.294133,
     "end_time": "2023-02-13T01:38:32.449994",
     "exception": false,
     "start_time": "2023-02-13T01:38:32.155861",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_ds = ShakespeareDataset(Config)\n",
    "val_ds = ShakespeareDataset(Config,is_test=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "95610f72",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:32.465802Z",
     "iopub.status.busy": "2023-02-13T01:38:32.464320Z",
     "iopub.status.idle": "2023-02-13T01:38:35.194926Z",
     "shell.execute_reply": "2023-02-13T01:38:35.193914Z"
    },
    "papermill": {
     "duration": 2.740921,
     "end_time": "2023-02-13T01:38:35.197597",
     "exception": false,
     "start_time": "2023-02-13T01:38:32.456676",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "lm = ShakespeareGPT(Config)\n",
    "lm = lm.to(device=Config.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ca759645",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:35.213426Z",
     "iopub.status.busy": "2023-02-13T01:38:35.212611Z",
     "iopub.status.idle": "2023-02-13T01:38:35.218312Z",
     "shell.execute_reply": "2023-02-13T01:38:35.217476Z"
    },
    "papermill": {
     "duration": 0.015961,
     "end_time": "2023-02-13T01:38:35.220303",
     "exception": false,
     "start_time": "2023-02-13T01:38:35.204342",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "optim = torch.optim.AdamW(lm.parameters(), lr=Config.lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "688cadf2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:35.234385Z",
     "iopub.status.busy": "2023-02-13T01:38:35.234112Z",
     "iopub.status.idle": "2023-02-13T01:38:35.238743Z",
     "shell.execute_reply": "2023-02-13T01:38:35.237869Z"
    },
    "papermill": {
     "duration": 0.013966,
     "end_time": "2023-02-13T01:38:35.240788",
     "exception": false,
     "start_time": "2023-02-13T01:38:35.226822",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def loss_fn(logits, targets):\n",
    "    B,T,C = logits.shape\n",
    "    logits = logits.view(B*T, C)\n",
    "    targets = targets.view(B*T)\n",
    "    loss = F.cross_entropy(logits,targets)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "796e7cf7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:35.254568Z",
     "iopub.status.busy": "2023-02-13T01:38:35.254312Z",
     "iopub.status.idle": "2023-02-13T01:38:35.264801Z",
     "shell.execute_reply": "2023-02-13T01:38:35.263979Z"
    },
    "papermill": {
     "duration": 0.019624,
     "end_time": "2023-02-13T01:38:35.266622",
     "exception": false,
     "start_time": "2023-02-13T01:38:35.246998",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_N_iters():\n",
    "    lm.train()\n",
    "    train_step_losses = []\n",
    "    for batch in tqdm(range(Config.train_iters)):\n",
    "        optim.zero_grad()\n",
    "        inputs, targets = train_ds.get()\n",
    "        inputs, targets = inputs.to(device=Config.device), targets.to(device=Config.device)\n",
    "        logits = lm(inputs)\n",
    "        loss = loss_fn(logits,targets)\n",
    "        loss.backward()\n",
    "        optim.step()\n",
    "        train_step_losses.append(loss.item())\n",
    "        \n",
    "        if batch%(Config.train_iters//10)==0 or batch==Config.train_iters-1:\n",
    "            print(f\"batch {batch} train step loss: {loss.item()}\")\n",
    "        \n",
    "        del inputs, targets, loss, logits\n",
    "        \n",
    "    return train_step_losses\n",
    "    \n",
    "@torch.no_grad()\n",
    "def valid_N_iters():\n",
    "    lm.eval()\n",
    "    val_step_losses = []\n",
    "    for batch in tqdm(range(Config.val_iters)):\n",
    "        inputs, targets = val_ds.get()\n",
    "        inputs, targets = inputs.to(device=Config.device), targets.to(device=Config.device)\n",
    "        logits = lm(inputs)\n",
    "        loss = loss_fn(logits,targets)\n",
    "        val_step_losses.append(loss.item())\n",
    "        \n",
    "        if batch%(Config.val_iters//10)==0 or batch==Config.val_iters-1:\n",
    "            print(f\"batch {batch} valid step loss: {loss.item()}\")\n",
    "        \n",
    "        del inputs, targets, loss, logits\n",
    "    \n",
    "    return val_step_losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "84d122b3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:35.281832Z",
     "iopub.status.busy": "2023-02-13T01:38:35.280329Z",
     "iopub.status.idle": "2023-02-13T01:38:35.285762Z",
     "shell.execute_reply": "2023-02-13T01:38:35.284917Z"
    },
    "papermill": {
     "duration": 0.014638,
     "end_time": "2023-02-13T01:38:35.287679",
     "exception": false,
     "start_time": "2023-02-13T01:38:35.273041",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def save_lm():\n",
    "    state_dict = lm.state_dict()\n",
    "    save_path = Path('./').resolve() / 'shakespeareGPT'\n",
    "    save_path.mkdir(exist_ok=True)\n",
    "    model_path = save_path / f'shakespeareGPT.pth'\n",
    "    torch.save(state_dict, model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d635ca11",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:35.302123Z",
     "iopub.status.busy": "2023-02-13T01:38:35.301346Z",
     "iopub.status.idle": "2023-02-13T01:38:35.306294Z",
     "shell.execute_reply": "2023-02-13T01:38:35.305403Z"
    },
    "papermill": {
     "duration": 0.01408,
     "end_time": "2023-02-13T01:38:35.308257",
     "exception": false,
     "start_time": "2023-02-13T01:38:35.294177",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_lm():\n",
    "    train_losses = train_N_iters()\n",
    "    valid_losses = valid_N_iters()\n",
    "    save_lm()\n",
    "    return train_losses, valid_losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5d0844a0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:38:35.322492Z",
     "iopub.status.busy": "2023-02-13T01:38:35.321658Z",
     "iopub.status.idle": "2023-02-13T01:47:05.944192Z",
     "shell.execute_reply": "2023-02-13T01:47:05.943112Z"
    },
    "papermill": {
     "duration": 510.631665,
     "end_time": "2023-02-13T01:47:05.946266",
     "exception": false,
     "start_time": "2023-02-13T01:38:35.314601",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad66758e50634016912e3a0e09db0efa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch 0 train step loss: 4.303605556488037\n",
      "batch 1000 train step loss: 2.454551935195923\n",
      "batch 2000 train step loss: 2.437880277633667\n",
      "batch 3000 train step loss: 2.315814256668091\n",
      "batch 4000 train step loss: 2.0116066932678223\n",
      "batch 5000 train step loss: 1.8777037858963013\n",
      "batch 6000 train step loss: 1.8346610069274902\n",
      "batch 7000 train step loss: 1.7588508129119873\n",
      "batch 8000 train step loss: 1.7117947340011597\n",
      "batch 9000 train step loss: 1.7134160995483398\n",
      "batch 9999 train step loss: 1.6846973896026611\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34bd37e964144ef8bad6429b394cad9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1000 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "batch 0 valid step loss: 1.7925341129302979\n",
      "batch 100 valid step loss: 1.7870755195617676\n",
      "batch 200 valid step loss: 1.8259053230285645\n",
      "batch 300 valid step loss: 1.7898788452148438\n",
      "batch 400 valid step loss: 1.7743006944656372\n",
      "batch 500 valid step loss: 1.7524681091308594\n",
      "batch 600 valid step loss: 1.8146271705627441\n",
      "batch 700 valid step loss: 1.8328163623809814\n",
      "batch 800 valid step loss: 1.769633412361145\n",
      "batch 900 valid step loss: 1.7880773544311523\n",
      "batch 999 valid step loss: 1.7822539806365967\n"
     ]
    }
   ],
   "source": [
    "tl,vl=train_lm()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a0be297",
   "metadata": {
    "papermill": {
     "duration": 0.00752,
     "end_time": "2023-02-13T01:47:05.961872",
     "exception": false,
     "start_time": "2023-02-13T01:47:05.954352",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Loss Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a7b79bd0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:47:05.978528Z",
     "iopub.status.busy": "2023-02-13T01:47:05.978231Z",
     "iopub.status.idle": "2023-02-13T01:47:06.211983Z",
     "shell.execute_reply": "2023-02-13T01:47:06.211002Z"
    },
    "papermill": {
     "duration": 0.244489,
     "end_time": "2023-02-13T01:47:06.214126",
     "exception": false,
     "start_time": "2023-02-13T01:47:05.969637",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAAqCUlEQVR4nO3deXxU1d3H8c+PEAg7IUREAgZc2WJYBBQroBYVXOtG3X2e1rq1RW2L2k0f20fb0tbiRrWP1hWkuFFrXSvihgoUKAjILgGBsCQQFoXkPH+cGzIkk2SSTDK5k+/79ZrX3OXce39nCL85c+6595pzDhERCb9miQ5ARETiQwldRCRJKKGLiCQJJXQRkSShhC4ikiSU0EVEkoQSuhxgZleb2fu13HakmeXFOyYRiZ0SehNjZieZ2YdmVmhm28zsAzM7PtFxhY2ZdTWzR81sg5kVmdkqM/urmR0brM82MxesKzKzNWZ2m5n1iFhWFJTZFTH/jSjHmmlm32n4WkrYNE90ANJwzKw98ApwPTANaAF8A/gqkXElmpk1d87tr0H5DODD4PUNYBXQATgf+CawNKJ4R+fcfjM7AXgbmO+caxuxLwcc55xbUfeaSFOnFnrTcjSAc26Kc67YObfHOfeGc25hZCEzm2hm281stZmdGbH8GjNbYmY7gxbp9yo7kJn9wMw+M7MsM2sZ7PMLM9tkZpPNrFVQrrOZvWJmBcEvhvfMrFmwbo2Z3R7sZ7uZPW5maRHHOMvM5gfbfmhmORHrbjOzlUGsn5nZ+RHrrg5+mfzRzLYBd1YVYxQ3AzuAK5xzK51X4Jx73Dl3f7QNnHMfAYuBfpV9ZjVlZs3M7GdmttbMNpvZk2bWIViXZmZPm9nW4PP51My6RNR/VfDZrDazyyL2+V/Bv/F2M3vdzA4PllvweW0Oft0tNLO41UXiQwm9afkcKDazJ8zsTDNLj1JmKLAM6Az8Fvg/M7Ng3WbgLKA9cA3wRzMbWH4HZvZz4GpghHMuD/gN/sskFzgS6Ab8Iih+K5AHZAJdgDuAyPtRXAacDhwR7ONnwTEGAo8B3wMygD8DM8ysZbDdSnzruQNwF/C0mXUtV89VwCHAr6uJsbzTgBedcyWVrC//eZiZDQf6Av+OZZsYXR28RgG9gLbAA8G6q/B1747/fK4D9phZG2AScKZzrh1wIjA/iPM8/Of/Lfy/x3vAlGB/o4GT8Z9RR+ASYGsc6yLx4JzTqwm9gN7AX/FJdD8wA+gSrLsaWBFRtjU+uR5ayb5eAn4YTI8E1gN/AN4HOgTLDdgFHBGx3QnA6mD6f4CXgSOj7H8NcF3E/BhgZTD9MHB3ufLL8F8i0WKdD5wbUc8vItZVGWOUfa0oF9c5QAGwE3gjWJYdfHYFwHZgCfCDKPty0epersxM4DtRlr8N3BAxfwywD9+V+l/4LqGcctu0CWK6AGhVbt0/gf+OmG8G7AYOB07BNwiGAc0S/XesV/SXWuhNjHNuiXPuaudcFv7n/2HAfRFFNkaU3R1MtgUIWvWzg66RAnyC7RyxbUfgWuAe51xhsCwT/8UwN/jpXwC8FiwH+B0+Qb4RdAPcVi7kdRHTa4N4wSeZW0v3Gey3e+l6M7syojumIKhrZKyR+60uxvK2Agda+865Gc65jviumBblynZ2zqU753o75yZVsr/aOgz/mZRai0/mXYCngNeBqeZP3P7WzFKdc7vwrevrgC/N7B8WnMjFf6Z/ivgMtuG/7Lo55/6Fb/0/CGwys0fMn5ORRkQJvQlzzi3Ft9ar7QsNujKeBybiW/QdgVfx/+FLbcd3yTwedDEAbAH2AH2dcx2DVwcXnBh0zu10zt3qnOsFnA3cYmanRuyze8R0D2BDML0O+HXEPjs651o756YE/b6PAjcBGUGsi8rFGtmtU2WMUbwNnFfa159AG/BJuFQP/K+uTc65fc65u5xzffDdKmcBVwI45153zn0T/6W0FP9Zgf9Mv1fuM23lnPsw2G6Sc24QvuvoaODHDVBHqYFE/0FKAzKzY83sVjPLCua7A98GZseweQugJZAP7Dd/snR0+ULOuZn4fu8XzWyo8/3Mj+L72w8JjtvNzE4Pps8ysyODfvodQHHwKnWj+ROrnfD9u88Fyx8FrjOzoUEfdRszG2tm7fDdCi6IFTO7hiq+tKqLMYo/AOnAU2Z2RHD8dvj+9/rSPDjRWfpKxfdv32xmPc2sLfC/wHPOj6oZZWb9zSwF/7nuw58/6WJm5wR96V8BRZR93pOB282sL4CZdTCzi4Lp44PPOhXfPbWXg/+dpBFQQm9aduJPBn5sZrvwiXwR/sRklZxzO4Ef4Ic7bgcuxfe/Ryv7Jv6k6QwzGwRMwHerzDazHcBb+P5egKOC+SLgI+Ch4Euh1LPAG/gTmKuAXwXHmAN8F98NsD3Y/9XBus+A3wf72wT0Bz6opopVxVi+flvwfcl78ecLduL76Nvhh4TWh4fxvyJKX4/jTwo/BcwCVgfxfD8ofygwHZ/MlwDvAk/j/8/fim/dbwNGADcE9XoRf3J4avAZLAJKRzm1x3/pbcd37WzF/1qTRsSc0wMupHEyszX4k4FvJToWkTBQC11EJEkooYuIJAl1uYiIJAm10EVEkkTCbs7VuXNnl52dnajDi4iE0ty5c7c456Je9JawhJ6dnc2cOXMSdXgRkVAys7WVrVOXi4hIklBCFxFJEkroIiJJQk8sEpG42rdvH3l5eezduzfRoYRaWloaWVlZpKamxryNErqIxFVeXh7t2rUjOzubsmejSE0459i6dSt5eXn07Nkz5u3U5SIicbV3714yMjKUzOvAzMjIyKjxrxwldBGJOyXzuqvNZxi+hF6wCBb8HPZuTnQkIiKNSvgS+o4lsPhXsDc/0ZGISCNUUFDAQw89VKttx4wZQ0FBQczl77zzTiZObDy3hQ9fQj/wFLGYHrguIk1MVQm9uLjqhyy9+uqrdOzYsR6iahjhTei6S6SIRHHbbbexcuVKcnNz+fGPf8zMmTMZNWoUl156Kf379wfgvPPOY9CgQfTt25dHHnnkwLbZ2dls2bKFNWvW0Lt3b7773e/St29fRo8ezZ49e6o87vz58xk2bBg5OTmcf/75bN++HYBJkybRp08fcnJyGDduHADvvvsuubm55ObmMmDAAHbu3BmXuodv2OKB5/IqoYs0enPHw/b58d1nei4Muq/S1ffeey+LFi1i/nx/3JkzZ/LJJ5+waNGiA0MAH3vsMTp16sSePXs4/vjjueCCC8jIyDhoP8uXL2fKlCk8+uijXHzxxTz//PNcfvnllR73yiuv5P7772fEiBH84he/4K677uK+++7j3nvvZfXq1bRs2fJAd87EiRN58MEHGT58OEVFRaSlpdXlEzkgxC10dbmISGyGDBly0HjuSZMmcdxxxzFs2DDWrVvH8uXLK2zTs2dPcnNzARg0aBBr1qypdP+FhYUUFBQwYsQIAK666ipmzZoFQE5ODpdddhlPP/00zZv7NvTw4cO55ZZbmDRpEgUFBQeW15Va6CJSf6poSTekNm3aHJieOXMmb731Fh999BGtW7dm5MiRUcd7t2zZ8sB0SkpKtV0ulfnHP/7BrFmzmDFjBnfffTeLFy/mtttuY+zYsbz66qsMGzaMt956i2OPPbZW+4+kFrqIJJV27dpV2SddWFhIeno6rVu3ZunSpcyePbvOx+zQoQPp6em89957ADz11FOMGDGCkpIS1q1bx6hRo/jtb39LQUEBRUVFrFy5kv79+zNhwgQGDx7M0qVL6xwDhLKFXjrKRS10EakoIyOD4cOH069fP84880zGjh170PozzjiDyZMnk5OTwzHHHMOwYcPictwnnniC6667jt27d9OrVy8ef/xxiouLufzyyyksLMQ5x80330zHjh35+c9/zjvvvENKSgp9+vThzDPPjEsMCXum6ODBg12tHnCx/lV4dyyM/hg6D4l/YCJSJ0uWLKF3796JDiMpRPsszWyuc25wtPLh63IxjUMXEYkmfAm9NGSNQxcROUj4Erpa6CIiUYUvoetKURGRqMKX0DUOXUQkqvAldI1DFxGJKnwJXS10EYmztm3bArBhwwYuvPDCqGVGjhxJtKHWlS1PhPAldLXQRaSeHHbYYUyfPj3RYdRa+BK6rhQVkSpMmDDhoPuh33nnnfz+97+nqKiIU089lYEDB9K/f39efvnlCtuuWbOGfv36AbBnzx7GjRtHTk4Ol1xySUz3cpkyZQr9+/enX79+TJgwAfD3YL/66qvp168f/fv3549//CMQ/ba6dRW+S/9Rl4tIWIwfD8FdbOMmNxfuu6/y9ePGjWP8+PHccMMNAEybNo3XXnuNtLQ0XnzxRdq3b8+WLVsYNmwY55xzTqXP7nz44Ydp3bo1CxcuZOHChQwcOLDKuDZs2MCECROYO3cu6enpjB49mpdeeonu3buzfv16Fi1aBHDgFrrRbqtbV+FtoavLRUSiGDBgAJs3b2bDhg0sWLCA9PR0evTogXOOO+64g5ycHE477TTWr1/Ppk2bKt3PrFmzDtz/PCcnh5ycnCqP++mnnzJy5EgyMzNp3rw5l112GbNmzaJXr16sWrWK73//+7z22mu0b9/+wD7L31a3rmLei5mlAHOA9c65s8qtM+BPwBhgN3C1c25eXCKsQFeKioRFVS3p+nThhRcyffp0Nm7ceKA745lnniE/P5+5c+eSmppKdnZ21NvmRqqs9R5NZffFSk9PZ8GCBbz++us8+OCDTJs2jcceeyzqbXXrmthr0kL/IbCkknVnAkcFr2uBh+sUVVV0paiIVGPcuHFMnTqV6dOnHxi1UlhYyCGHHEJqairvvPMOa9eurXIfJ598Ms888wwAixYtYuHChVWWHzp0KO+++y5btmyhuLiYKVOmMGLECLZs2UJJSQkXXHABd999N/Pmzav0trp1FdPXgZllAWOBXwO3RClyLvCk819Rs82so5l1dc59WecIK0bj39RCF5FK9O3bl507d9KtWze6du0KwGWXXcbZZ5/N4MGDyc3NrfaBEtdffz3XXHMNOTk55ObmMmRI1Xd37dq1K/fccw+jRo3COceYMWM499xzWbBgAddccw0lJb4Res8991R6W926iun2uWY2HbgHaAf8KEqXyyvAvc6594P5t4EJzrk55cpdi2/B06NHj0HVfUNGtfVTeH0IjHgFuo2tvryINCjdPjd+4n77XDM7C9jsnJtbVbEoyyp8UzjnHnHODXbODc7MzKzu0FUfSidFRUQOEksf+nDgHDNbA0wFTjGzp8uVyQO6R8xnARviEmF5ulJURCSqahO6c+5251yWcy4bGAf8yzl3ebliM4ArzRsGFNZP/zmohS7S+CXqSWjJpDafYa3HyJjZdcFBJwOv4ocsrsAPW7ymtvuN4cDBhP5gRBqjtLQ0tm7dSkZGRo2G/UkZ5xxbt24lLS2tRtvVKKE752YCM4PpyRHLHXBjjY5ca+pyEWnMsrKyyMvLIz8/P9GhhFpaWhpZWVk12iZ8l/7rSlGRRi01NZWePXsmOowmKXyX/quFLiISVfgSulroIiJRhS+h60pREZGowpfQNQ5dRCSq8CV0jUMXEYkqfAldLXQRkajCl9DVQhcRiSp8CV1XioqIRBW+hK5x6CIiUYUvoWscuohIVOFL6Gqhi4hEFb6Erha6iEhU4Uvo6KSoiEg04UvopePQdem/iMhBwpfQD7TQ1eUiIhIpfAldLXQRkajCl9B1paiISFThTeg6KSoicpDwJXTdnEtEJKoQJnR1uYiIRBO+hK4rRUVEogpfQlcLXUQkqvAldJ0UFRGJKnwJXePQRUSiCl9C15WiIiJRhS+hq4UuIhJV+BK6WugiIlFVm9DNLM3MPjGzBWa22MzuilJmpJkVmtn84PWL+gkXyi79VwtdRCRS8xjKfAWc4pwrMrNU4H0z+6dzbna5cu85586Kf4jl6EpREZGoqk3ozjkHFAWzqcErgdlU49BFRKKJqQ/dzFLMbD6wGXjTOfdxlGInBN0y/zSzvpXs51ozm2Nmc/Lz82sXsVroIiJRxZTQnXPFzrlcIAsYYmb9yhWZBxzunDsOuB94qZL9POKcG+ycG5yZmVm7iHWlqIhIVDUa5eKcKwBmAmeUW77DOVcUTL8KpJpZ5zjFWJGlgCuut92LiIRRLKNcMs2sYzDdCjgNWFquzKFmvulsZkOC/W6Ne7QHDtgcSvbV2+5FRMIollEuXYEnzCwFn6inOedeMbPrAJxzk4ELgevNbD+wBxgXnEytH81Swe2vt92LiIRRLKNcFgIDoiyfHDH9APBAfEOrgjWHEiV0EZFIIbxSFGjWHJy6XEREIoU0oaeqhS4iUk44E7o1Vx+6iEg54U3oGuUiInKQcCZ0jXIREakgpAldo1xERMoLZ0I3jXIRESkvnAldo1xERCqI5UrRxmfb3ERHICLS6ISzhS4iIhUooYuIJIlwdrlknQdFKxMdhYhIoxLOFnpKGhR/legoREQalRAn9L2JjkJEpFEJZ0Jv1hJKlNBFRCKFM6EX74G9mxMdhYhIoxLOhL76Sf++bV5i4xARaUTCmdBL7duZ6AhERBqNcCd0nRgVETkgnAn9+If9e6tDExuHiEgjEs6E3rq7fy/5OrFxiIg0IuFM6M1S/bsSuojIASFN6C38ux5DJyJyQEgTulroIiLlhTShBy103c9FROSAcCb0lDT/XqKELiJSKpwJvVlL/75zRWLjEBFpREKa0IMulwW3JzYOEZFGpNqEbmZpZvaJmS0ws8VmdleUMmZmk8xshZktNLOB9RNu6QHD+T0kIlKfYnli0VfAKc65IjNLBd43s38652ZHlDkTOCp4DQUeDt7rR/M29bZrEZGwqrap67yiYDY1eLlyxc4FngzKzgY6mlnX+IYaoWWGfz98XL0dQkQkbGLquzCzFDObD2wG3nTOfVyuSDdgXcR8XrCs/H6uNbM5ZjYnPz+/liFHWDu17vsQEUkSMSV051yxcy4XyAKGmFm/ckUs2mZR9vOIc26wc25wZmZmjYMVEZHK1ejsonOuAJgJnFFuVR7QPWI+C9hQl8BERKRmYhnlkmlmHYPpVsBpwNJyxWYAVwajXYYBhc65L+Md7EF6XATte9frIUREwiSWUS5dgSfMLAX/BTDNOfeKmV0H4JybDLwKjAFWALuBa+op3jK71sKOJfV+GBGRsKg2oTvnFgIDoiyfHDHtgBvjG1o1tn5SenCwaF34IiJNS/iv0Nm9rvoyIiJNQPgT+vb5iY5ARKRRSIKEviDREYiINArhTehdTvHvX0xLbBwiIo1EeBN6zyv9e9Y5iY1DRKSRCG9CTz8umAhvFURE4im82dAV+/fFv0psHCIijUR4E/q+HYmOQESkUQlvQj9kRKIjEBFpVMKb0PXUIhGRgyRHVvxqW6IjEBFJuORI6F9vT3QEIiIJlxwJfe74REcgIpJw4U7opc8W3fBKYuMQEWkEwp3QDz090RGIiDQa4U7oA3+f6AhERBqNcCd0SymbLv46cXGIiDQC4U7ouLLJeTcnLgwRkUYg3Am9ZWbZ9PKHEheHiEgjEO6EbgZdTi2b13h0EWnCwp3QAUbMKJvePCtxcYiIJFj4E3rz1mXTs85LWBgiIokW/oRe3ub3Ex2BiEhCJEdC/2ZEEn/rG1BSnLhYREQSJDkSeubwg+enNtcDMESkyUmOhA4w8L6D5//WAT6+NiGhiIgkQvIk9GN/WHHZykfhWYPirxo+HhGRBpY8CR1g7GfRlz+XBh98G5ZPBudg8T2wa13DxiYiUs+qTehm1t3M3jGzJWa22MwqNIXNbKSZFZrZ/OD1i/oJtxodesPZK6KvWzsVPr0epjSDBXfAyz3g3z+BnZWUFxEJmVha6PuBW51zvYFhwI1m1idKufecc7nB63/iGmVNtDsCzlkdW9klv4O/HwXz74Bda/2yHcvhs9/BkomwZ2P9xSkiEmfNqyvgnPsS+DKY3mlmS4BuQCX9G41A22zI/Q0s/DmUxHAXxs/u8a/y/v1jOPPfkNYVWnU5eN2O5bD7Czj01IrbiYgkgDnnqi9VWtgsG5gF9HPO7YhYPhJ4HsgDNgA/cs4tjrL9tcC1AD169Bi0du3aOoQeo3+dDhvfiN/+0g6FNj1g6yd+/tSZsHcTHH6xH/9uzfw9Zsr7ahvs3wmp7aFFul+2fw8sfxCOGQ/Nqv1uFRHBzOY65wZHXRdrQjeztsC7wK+dcy+UW9ceKHHOFZnZGOBPzrmjqtrf4MGD3Zw5c2I6dlwsvgdW/gWKVjXcMQGOuh7a9vKt/VKnvA3Fe+HdsX7+6O/D4Ek+wTdLrZjcXQm4Yr9ORJq0Oid0M0sFXgFed879IYbya4DBzrktlZVp8IQeaccy+PINWHQXfLU1MTFU55CToVkL/xCPL18vW57zK+j307J553zCL94NKa3U0hdJcnVK6GZmwBPANufc+ErKHApscs45MxsCTAcOd1XsPKEJPdJX26B5G1h2HxzxXZg5BrZ+nOioau/EKdCiI7Q/GlodBilpiY5IROKorgn9JOA94D9ASbD4DqAHgHNuspndBFyPHxGzB7jFOfdhVfttNAk9mmcNWmdB9wuh02Dofh5Ma5voqGqn1zVw5LWw8W3oeTm0OTzREYlIHcSlDz3eGnVC37vZd1+ktqu4bnceLPk9DJgIBH3be76E/A/goyt8mSGPwCfXQqdBsG1ug4ZerdT20O5o36/fIh2Of8ifyBWRUFBCbwyc8ydCi3dDywy/7IUu/ssj0c79Atp0T3QUIhKDqhK6zqA1FDNo3sq/Sn1rU8VyWz6Gkn1QsMBfxdrnJ/DF87Bjib/4qXU3WPEInPIWrH4KVj9R99he7uHfL9518ANDRCRU1EIPu7dPhazzIOsc3z++fQHMuxk2vVO7/Z0xx3cViUijpC6XpmjPJn91645l0Cbb9/VPaxP79qe8DYeeUm/hiUjtVJXQdTYsWZXeqqD9MZDS0nelXOr8a8xCaJlZ9fb/OhXWveRH/Hy9vd7DFZG6U0Jvijr2hws2R+/Dj/Te+f59+4L6j0lE6kwJvSlLOwTO/xL63wnH/brycm+PgkVVrBeRRkF96HKw3RvgpW7R112yx7/r6lORhFEfusSu9WGQe2/0dc+18i8RaZSU0KWiPhPghCcrX/+sQcGihotHRGKihC7R9bwCvl1c+fq1UxsuFhGJiRK6VM6awblrYMTfK65b/OuD7/EuIgmnhC5Va3M4dDsr+rolE2HXuoaNR0QqpYQusTl/A5yzsuLyl3vAK8f6K1JFJKGU0CU2rbr6W+5e8lXFdTuW+aRevLfh4xKRA5TQpWZSWsBhY6Ov26rrCkQSSQldam7kKzAwyqNl3/qGH9LoSiquE5F6p4QutXPszZWvm54OS//YcLGICKCELnVxYQFYSsXl+3bAvFtg+Z9hy+wGD0ukqVJCl9pr0QHGfQ0d+kRf/+l18MYJsHOlfwSfiNQrJXSpG2sGYxfD2SsqL/P3I2FKM/Wti9QzJXSJj3ZHQPveVZeZcYR/XqqI1AsldImfsz7z91c/e3n09bvWwNQWsH9Pg4Yl0lQooUt8tToU2h1ZdZlprfVYO5F6oIQu9aPX1VWvn97Jj1lfO61BwhFpCpTQpX4cPxlO/wTaHVV1uQ8uCR5EXQib39doGJE6UEKX+pHSEjKOh7M/h36/rL789I7+StPPH4C8v/ux7CJSI3qmqDScvZvhhS6xl888Cb75Xv3FIxJCdXqmqJl1N7N3zGyJmS02sx9GKWNmNsnMVpjZQjMbGI/AJcmkHQIXF0H3b8VWPv99WPWkxq+LxCiWLpf9wK3Oud7AMOBGMyt/aeCZwFHB61rg4bhGKcmjeRsY9tfYy8++Cqak+H72fw6AXWvrLTSRsKs2oTvnvnTOzQumdwJLgG7lip0LPOm82UBHM+sa92glOaS2g0udv23A4Adi3277fHg5G9a/AoWf1Vd0IqFVo5OiZpYNDAA+LreqGxD5LLI8KiZ9zOxaM5tjZnPy8/NrGKoknWapcPSNPrlfWBD7du+eDf/oC1/XYBuRJiDmhG5mbYHngfHOufJDECzKJhXOtjrnHnHODXbODc7MzKxZpJLcWnSAo26o2TbT0+HfP/HTJcXw+UNQHOWJSiJNRPNYCplZKj6ZP+OceyFKkTyge8R8FrCh7uFJk3L8g3Dk92DFn6FZS1gWwz3Vl/zOv0rNuRFOmgaZJ8Omf4Erho79If24+otbpJGodtiimRnwBLDNOTe+kjJjgZuAMcBQYJJzbkhV+9WwRYnJ7g3wUoXeu5o7Yx50GlD3/YgkWFXDFmNpoQ8HrgD+Y2bzg2V3AD0AnHOTgVfxyXwFsBu4po4xi3itD4MxC+HVnLrt57WIkbRtj4B2R8MJf4UW6b4vXyQJ6MIiCYfdG/x91U+bBZ/eANs+jc9+e1wCA34DX20BDDrpEgpp3KpqoSuhS/iU7IMPr4Avnov/vi/YAnkvQ/bl/vF6K/4Ma5+F/A/gop3QrLm/0Kl56/gfWyQGdbpSVKTRaZYKJ02F04LbAnQaFL99P98ZPv5vmNEL3hntT7Lmf+DX7VkPz7WCaW1g8b1Qst+/9u2suJ+9wbDcXet8GZEGoBa6JI/9u32yTYScu6HLKZAxFF44BL7eBrm/gfkT4JgfwqD7EhOXJB11uUjTsa8I/tYO+v4U9nwJqx5LdET+BOzJL8LWOZB9qe+2EamlJtflcsEFcOih8JOfQL9+sGZN2brly6GwEH72M9hf7pfw22/D7t0NGqrEW2pbuGSPbzEP+z9/FeqlDlpmJC6mnZ/7K1tnXwVTU/0j+Da/77tvNrwOBYugcEni4pOkEcoWelERnHQSnHoq3HMPPP88jBsHFlyvalGuWy0uhpkz/TYpKX7+mGPgoovgrrtgzhwYOtSX/cMf4Ac/gPx8GD0asrNh4kS47z64/36/vYTMvh1Q/LW/IrVwCXy9FeZ8HwoXJzqyg416A9K6wKzz4Njx0PMKP7Sy1AeXQeehcMwPEhWhJFhSdbmUlERPqNOnw6JFcOyxPrnXxMCBMG9e1WW6dIFNm3ziHxTHc3DSSCybBOkDYO542F7NH0Nj0e5oOOEpSM/18ykt/Pvm96D9sZBWze01XIm/VULzVvUapsRXUiX0Bx+Em26quPy442DBgjgEFgM9JS3J7d8FJV+DpcLCn8MX02BPCO5k0e0cWD+jbD7tUDjvi7I/2NKEX+rFLD9y5+Iifx+c7t+CNc+CNfNfbs1bQZdR1R/36wL/mbWOwxW9Uq2kSug33eSTeiIpoTdBzwb9eBftgD0b4ZWj/fxR18PyEN3+f8Q/4N2xvmX/0RWxb3f+RmhV7mlTXxcCJfD3o+CrrTBun0/uLTro6tt6lFQJvWfPg09yJoISehP0rEFqB7iowM9/MR2K9/g+bvAjWF4/3k8feS2seCQhYdarQX+CuRUeWFamWQv/y6btEXDi09C8nT9J3eZwfy/7z34La6f4k9Tgz2mAT/4b34JDRlT8FQH+P1y0E2NNVFIl9FGj/MnNRFJCb4LyP4Q22f7eMtGUDpds1RXO3wDFe2HWt6BlJz8efe9GmHmWf29qMobB1tkHLxv9EbxxQvTy7Y+BPrdDr6sg/yN480T/S+j4hyo/xpqp/sugqscbuhJY/SQcfmn0L46QSKqEvmoVHHFEPQRUA0roUivFe2F6J/8IvtZZsGQirP87ZJ4Im2f5Mv1+CYvuSmiYjcbRP4DPJ5XNn5cHKWl+COrqZ/yXZcZQ/36gS6wQUtv76ZJ9ULQG2vaCbXOhaBV8+G3/Gff+Ebx3gf+SaBdjQtnzJbToBCkt41rNmkqqhF5QAOnp1RarN9ddBw+HqMtUQmLPRtj6CWSdA6ufhszhPgm9f1FZmdM/LevWkdrr0LdsuGqbnnDuKj/996Nh53IY+hj0uAh2LIVdq/2N4eaNL9u+948g+zI/nLRFBuS95Ocju4VK8+ruPGgT+aiIukuqhA4N2512883wx+A5Cz/9KfzqVw13bBHAt+z37fTDEJc9AIed4Vudq5/yD+9440Qo+cr3ce9YBism++6FYY/DbN3JukH0ugaGPQafXOe71fJeLlt3xhx/T589G2DT27DuBbh4d62HiyZdQgdo1gz+8hc/5vzWW+Hii+GGG2DpUti3z5d5/XXo3Bn++leYPNkv++QTGDLEj1s//XRo0cK/oOyLYsUK362zZg0cfrgf+75unb/ASKTRWTsN/vNLGLvYDzmM9O45vlsn9zfQ5ye+C2JGz4SEKRF6XgknPFGrTZMyoUdTUuKTecsoXVxFRdCmjU/au3b56fIKC+Gdd+C88+IalkjifLXNj6M/8ntlLZa8GTDrXDjuf2HBHZD5Dch/Dw4d7R/ZN/QRX2bezb78UTf4+8R//J3E1SMZXVq73NtkErqIxGjH59D+aN+F0+NCaJnpW/eR/ZmzvgWp7XxLcv9u38c87HF/sZE1889szTwZlt3n7yp5ylv+tgpdT4eNb/pbD0fKvsLfmGzV4w1a1UZLCV1EGh3nfN99h2PLln293Y/oSTsETp/j+/9T0vy6v6VD/1/CYWMh//3gCtWnYU7EJeAZQ/xJ4s4n+pOPG/7RsHVqCPWQ0HUfTxGpG7ODkzn4JDxoEnQ7u+Ioj4u2l023P8q/H30jHPFdeK6lfyzgSVMP3uZZg1aHwfnr/fyKR+Hz+2HUm35MedFqeHO4P4EM0LwN9P8f+PwBP1KliVALXUQav7yX/ZOpWmdVXmZvvn+wSKuukNK67L7zxXv9fXm+mOa/YFLbwq61fkTK0EfhpYgvnIyh/uZsF2yDddN9+ec7l63v/i0/SqX3j/x1BLV1+qeQEbWRXS210EUk3LLOrb5MWmb0O0yWdvVkf7tsWZvDYdQ//fTRN/mEP+gPB2/X62r/3vcOf8/6k6YdfFFR0Rqf9Ac/AK26+S6jYY9D12/Cxn/5spnDYflk+PT6su2O+99aJ/PqqIUuIlIbXxfChlcP/qKozNZPYc0zMGCif/h4HS6mUQtdRCTeWnSILZkDZBzvX/UsKR9BJyLSFCmhi4gkCSV0EZEkoYQuIpIklNBFRJKEErqISJJQQhcRSRJK6CIiSSJhV4qaWT6wtpabdwa2xDGcMFCdmwbVuWmoS50Pd85FucdBAhN6XZjZnMoufU1WqnPToDo3DfVVZ3W5iIgkCSV0EZEkEdaE/kiiA0gA1blpUJ2bhnqpcyj70EVEpKKwttBFRKQcJXQRkSQRuoRuZmeY2TIzW2FmtyU6ntoys+5m9o6ZLTGzxWb2w2B5JzN708yWB+/pEdvcHtR7mZmdHrF8kJn9J1g3yawOj0NpAGaWYmb/NrNXgvmkrrOZdTSz6Wa2NPj3PqEJ1Pnm4O96kZlNMbO0ZKuzmT1mZpvNbFHEsrjV0cxamtlzwfKPzSy72qCcc6F5ASnASqAX0AJYAPRJdFy1rEtXYGAw3Q74HOgD/Ba4LVh+G/CbYLpPUN+WQM/gc0gJ1n0CnAAY8E/gzETXr5q63wI8C7wSzCd1nYEngO8E0y2AjslcZ6AbsBpoFcxPA65OtjoDJwMDgUURy+JWR+AGYHIwPQ54rtqYEv2h1PADPAF4PWL+duD2RMcVp7q9DHwTWAZ0DZZ1BZZFqyvwevB5dAWWRiz/NvDnRNeninpmAW8Dp1CW0JO2zkD7ILlZueXJXOduwDqgE/4xl68Ao5OxzkB2uYQetzqWlgmmm+OvLLWq4glbl0vpH0qpvGBZqAU/pQYAHwNdnHNfAgTvhwTFKqt7t2C6/PLG6j7gJ0BJxLJkrnMvIB94POhm+ouZtSGJ6+ycWw9MBL4AvgQKnXNvkMR1jhDPOh7Yxjm3HygEMqo6eNgSerT+s1CPuzSztsDzwHjn3I6qikZZ5qpY3uiY2VnAZufc3Fg3ibIsVHXGt6wGAg875wYAu/A/xSsT+joH/cbn4rsWDgPamNnlVW0SZVmo6hyD2tSxxvUPW0LPA7pHzGcBGxIUS52ZWSo+mT/jnHshWLzJzLoG67sCm4PlldU9L5guv7wxGg6cY2ZrgKnAKWb2NMld5zwgzzn3cTA/HZ/gk7nOpwGrnXP5zrl9wAvAiSR3nUvFs44HtjGz5kAHYFtVBw9bQv8UOMrMeppZC/yJghkJjqlWgjPZ/wcscc79IWLVDOCqYPoqfN966fJxwZnvnsBRwCfBz7qdZjYs2OeVEds0Ks65251zWc65bPy/3b+cc5eT3HXeCKwzs2OCRacCn5HEdcZ3tQwzs9ZBrKcCS0juOpeKZx0j93Uh/v9L1b9QEn1SoRYnIcbgR4SsBH6a6HjqUI+T8D+fFgLzg9cYfB/Z28Dy4L1TxDY/Deq9jIiz/cBgYFGw7gGqOXHSGF7ASMpOiiZ1nYFcYE7wb/0SkN4E6nwXsDSI9yn86I6kqjMwBX+OYB++Nf3f8awjkAb8DViBHwnTq7qYdOm/iEiSCFuXi4iIVEIJXUQkSSihi4gkCSV0EZEkoYQuIpIklNBFRJKEErqISJL4fw+lqWhFruLPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tl,label='train loss',color='orange')\n",
    "plt.plot(vl,label='valid loss',color='blue')\n",
    "plt.title('Shakespeare GPT Losses')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2055dadb",
   "metadata": {
    "papermill": {
     "duration": 0.007849,
     "end_time": "2023-02-13T01:47:06.230383",
     "exception": false,
     "start_time": "2023-02-13T01:47:06.222534",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Generating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7c875717",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:47:06.247926Z",
     "iopub.status.busy": "2023-02-13T01:47:06.247101Z",
     "iopub.status.idle": "2023-02-13T01:47:17.229191Z",
     "shell.execute_reply": "2023-02-13T01:47:17.227767Z"
    },
    "papermill": {
     "duration": 10.992982,
     "end_time": "2023-02-13T01:47:17.231271",
     "exception": false,
     "start_time": "2023-02-13T01:47:06.238289",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "generated (100 tokens)\n",
      "==================================================\n",
      "\n",
      "HERS OF YORK:\n",
      "Iffer these peak than all for the be me;\n",
      "That thou, follived of frome, my does,\n",
      "I day;\n",
      "==================================================\n",
      "\n",
      "\n",
      "generated (300 tokens)\n",
      "==================================================\n",
      "\n",
      "LADY ANTIO:\n",
      "Now may hight. The is graclike a heavent;\n",
      "Ye'lr, or borse to your vales stend\n",
      "Hence meen shallessile to see ton them oth to:\n",
      "Goods!\n",
      "State, what shure gue and nay ild But or this it,\n",
      "Which cafter, more enhere making werefy loved bein;\n",
      "And in by the peop, by any home whyserving fried as Du\n",
      "==================================================\n",
      "\n",
      "\n",
      "generated (500 tokens)\n",
      "==================================================\n",
      "\n",
      "Havilse, my statings unto a havisbale of him it my seechies:\n",
      "Lords come, the must that calls body Greputing me;\n",
      "Twit son and dayped\n",
      "I cloody, but and not is\n",
      "And he dime my fire of us good brimsence to-phe dust.\n",
      "\n",
      "VOLEONTES:\n",
      "O, he here, out of; Camine thee!\n",
      "Become, I'll place, if that the kindly;\n",
      "The morrown hold you time fathere pass and forwords for in duke\n",
      "And I hadly me;--\n",
      "Whose well. I shall no lord shangelown my breation.\n",
      "\n",
      "Lood\n",
      "Say, that?\n",
      "\n",
      "DUKE VINCENTIO:\n",
      "Yet should kind prost, you speak\n",
      "To \n",
      "==================================================\n",
      "\n",
      "\n",
      "generated (700 tokens)\n",
      "==================================================\n",
      "\n",
      "Warward from ant, the gland'd, Cland. What stand\n",
      "Hy good it which dest, in aface,\n",
      "Cut it in He he have say crown.\n",
      "So time say, the the of shall'd the is,\n",
      "To must our thince take morrow,\n",
      "Whost none that pare of donessent sons,\n",
      "To sweet senaives an Edward with and blain.\n",
      "No we sir, but my painter:\n",
      "Go have in\n",
      "Theress hingstirges speak you sweed on in be nothe look your elf!\n",
      "Those sir, and cansweet day.\n",
      "When how condesputy, place, with day of wodds! I than me,\n",
      "Mineds neven hou his fore hight is to trumerchedgen's will\n",
      "From fair.\n",
      "\n",
      "JULIET:\n",
      "And we implead, love des, theaven per;\n",
      "He bear\n",
      "Eared that which ful the condeasion and to kinder bate?\n",
      "\n",
      "GLOUCESTER:\n",
      "Now a iz?\n",
      "\n",
      "PARITLBOLINA:\n",
      "Have York, by orth,\n",
      "==================================================\n",
      "\n",
      "\n",
      "generated (1000 tokens)\n",
      "==================================================\n",
      "\n",
      "Could as her hers I have him-kill obed of and harm\n",
      "Shell bear, such seemforatroud doyarriend about their;\n",
      "But the lund one table that dide my some way;\n",
      "And trun screnate that a had heachet!\n",
      "\n",
      "VIRCINIUS:\n",
      "That is your his be so day, what I lord; forrongue.\n",
      "Jurder mame duke this the pars!\n",
      "\n",
      "MENENIUS:\n",
      "Hap as out I to jury that'st me you?\n",
      "\n",
      "VOLIO:\n",
      "Vick,\n",
      "So that the comfor, see in man as brother ward, and:\n",
      "That is most serve to last our let not, and spoar\n",
      "A fll that Dukess'd the his me a welcound me\n",
      "Lihat mayin shave; the marish meel'd.\n",
      "Let your him distir, when you your be yound keech not\n",
      "Master.\n",
      "\n",
      "GLUMNIA:\n",
      "Which stoo, my let quest for to by it of for they old, whomies are!\n",
      "A brouch that conderal sace in;\n",
      "That have king oath, true; the have bond,\n",
      "And who scrught them to bedted thou trew and with my priseroof Rid, and rewant,\n",
      "But gon As and of Pome,\n",
      "The chars, and oft thing as appresse minitiar may bols? King as lords own.\n",
      "\n",
      "LADY GALET:\n",
      "For thid?\n",
      "I good ships wever of he vigul with the desphicher\n",
      "==================================================\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "generated_texts = []\n",
    "for length in [100,300,500,700,1000]:\n",
    "    generated = lm.generate(\n",
    "    torch.zeros((1,1),dtype=torch.long,device=Config.device), # initial context 0\n",
    "    total=length\n",
    ")\n",
    "    generated = tokenizer.decode(generated[0])\n",
    "    text=f'generated ({length} tokens)\\n{\"=\"*50}\\n{generated}\\n{\"=\"*50}\\n\\n'\n",
    "    generated_texts.append(text)\n",
    "    print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "52beb58c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-02-13T01:47:17.251205Z",
     "iopub.status.busy": "2023-02-13T01:47:17.249764Z",
     "iopub.status.idle": "2023-02-13T01:47:17.255447Z",
     "shell.execute_reply": "2023-02-13T01:47:17.254599Z"
    },
    "papermill": {
     "duration": 0.017286,
     "end_time": "2023-02-13T01:47:17.257706",
     "exception": false,
     "start_time": "2023-02-13T01:47:17.240420",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('generated.txt','w') as f:\n",
    "    for text in generated_texts:\n",
    "        f.write(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0b02019",
   "metadata": {
    "papermill": {
     "duration": 0.00793,
     "end_time": "2023-02-13T01:47:17.273822",
     "exception": false,
     "start_time": "2023-02-13T01:47:17.265892",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "---\n",
    "\n",
    "Full code: [shakespeareGPT](https://github.com/shreydan/shakespeareGPT)\n",
    "\n",
    "Follow [@shreydan](https://kaggle.com/shreydan) for more :)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 535.715335,
   "end_time": "2023-02-13T01:47:18.304464",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-02-13T01:38:22.589129",
   "version": "2.3.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "004ad0cec0bc46d5907f5ccd9722f0ea": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_8622c1c200f34d6bb4b987a76d92bd4f",
       "placeholder": "​",
       "style": "IPY_MODEL_4c9d5adacd8148fea6ea88995d596d0e",
       "value": "100%"
      }
     },
     "050ddef8ce7b4e51aa628160253599ca": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_23a1b820f6154bffb6803acc9e75c3a3",
       "placeholder": "​",
       "style": "IPY_MODEL_e78bde1733de458f9413912c6d13afae",
       "value": " 1000/1000 [00:14&lt;00:00, 67.58it/s]"
      }
     },
     "1e9aaf10f1694bb280bd79d9f3db165d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "23a1b820f6154bffb6803acc9e75c3a3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "3165a9bb0e3244628205cdbf583f6bd5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "34bd37e964144ef8bad6429b394cad9c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_004ad0cec0bc46d5907f5ccd9722f0ea",
        "IPY_MODEL_4b98714b562f4445b2c8d9f340a0add6",
        "IPY_MODEL_050ddef8ce7b4e51aa628160253599ca"
       ],
       "layout": "IPY_MODEL_61d27ad6638644da86baf5103fcadd65"
      }
     },
     "369c2689ba1f486782a85247370be043": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "4b98714b562f4445b2c8d9f340a0add6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7c8e349861074c38b066c0ee0d92ef1d",
       "max": 1000.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_369c2689ba1f486782a85247370be043",
       "value": 1000.0
      }
     },
     "4c9d5adacd8148fea6ea88995d596d0e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "61d27ad6638644da86baf5103fcadd65": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "63836e0988bd4a3d9e8ac467312ebddf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_9fa8de0974c94e5d8044240644e85a0e",
       "max": 10000.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_1e9aaf10f1694bb280bd79d9f3db165d",
       "value": 10000.0
      }
     },
     "7c8e349861074c38b066c0ee0d92ef1d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "83b282ee4ca447edbaf56784ad171d58": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8622c1c200f34d6bb4b987a76d92bd4f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9b147882b24248ec81b9f25434bac569": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e9f60ed37e0c4344b6afcd93f016393e",
       "placeholder": "​",
       "style": "IPY_MODEL_e0bb57b3f1e74306b223cee7ff397f3a",
       "value": "100%"
      }
     },
     "9fa8de0974c94e5d8044240644e85a0e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ad66758e50634016912e3a0e09db0efa": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_9b147882b24248ec81b9f25434bac569",
        "IPY_MODEL_63836e0988bd4a3d9e8ac467312ebddf",
        "IPY_MODEL_cdc356764c8b4f2ea9b9c0127a6bcf59"
       ],
       "layout": "IPY_MODEL_3165a9bb0e3244628205cdbf583f6bd5"
      }
     },
     "cdc356764c8b4f2ea9b9c0127a6bcf59": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_83b282ee4ca447edbaf56784ad171d58",
       "placeholder": "​",
       "style": "IPY_MODEL_d22ecc5fa6c141abb52f95160f3611ae",
       "value": " 10000/10000 [08:15&lt;00:00, 20.39it/s]"
      }
     },
     "d22ecc5fa6c141abb52f95160f3611ae": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "e0bb57b3f1e74306b223cee7ff397f3a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "e78bde1733de458f9413912c6d13afae": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "e9f60ed37e0c4344b6afcd93f016393e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}