tugda_da_example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "id": "X_QBOBKvo03o",
    "outputId": "091cd977-c04d-40a6-ccaf-fe7a4b0b1a95"
   },
   "source": [
    "### TUGDA on Domain Adaptation settings (PDX)\n",
    "In this notebook we present how TUGDA on Domain Adaptation settings (figure 1, preprint) can be trained using the GDSC (*in vitro* settings) and PDX (*in vivo*) datasets. Here we trained and evaluated TUGDA on set of best hyperparameters (previously found on validation data). Results reproduced (figure 3) using Pytorch (1.5.1, cuda 10.1) on GPU settings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "#install required libs\n",
    "!pip install pandas\n",
    "!pip install torch==1.5.1+cu101  -f https://download.pytorch.org/whl/torch_stable.html\n",
    "#in this version we require pytorch-lightning==0.9.0\n",
    "!pip install pytorch-lightning==0.9.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import random\n",
    "from itertools import cycle\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from torch.autograd import Function\n",
    "import pytorch_lightning as pl\n",
    "from pytorch_lightning import Callback\n",
    "from pytorch_lightning import seed_everything"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#flip discriminator gradient;\n",
    "class ReverseLayerF(Function):\n",
    "\n",
    "    @staticmethod\n",
    "    def forward(ctx, x, alpha):\n",
    "        ctx.alpha = alpha\n",
    "\n",
    "        return x.view_as(x)\n",
    "\n",
    "    @staticmethod\n",
    "    def backward(ctx, grad_output):\n",
    "        output = grad_output.neg() * ctx.alpha\n",
    "\n",
    "        return output, None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MetricsCallback(Callback):\n",
    "    \"\"\"PyTorch Lightning metric callback.\"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.metrics = []\n",
    "\n",
    "    def on_test_end(self, trainer, pl_module):\n",
    "        self.metrics.append(trainer.callback_metrics)\n",
    "\n",
    "    def on_train_end(self, trainer, pl_module):\n",
    "        self.metrics.append(trainer.callback_metrics)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class tugda_da(pl.LightningModule):\n",
    "    def __init__(self, params, train_data, y_train, x_target_unl,\n",
    "                 test_data, y_test, y_test_da\n",
    "                ):\n",
    "        super(tugda_da, self).__init__()\n",
    "        \n",
    "        self.learning_rate = params['lr']\n",
    "        self.batch_size = params['bs_source']\n",
    "        self.mu = params['mu']\n",
    "        self.lambda_ = params['lambda_']\n",
    "        self.gamma = params['gamma']\n",
    "        self.lambda_disc = params['lambda_disc']\n",
    "        self.bs_disc = params['bs_disc']\n",
    "        self.n_epoch = params['n_epochs']\n",
    "        self.passes = params['passes']\n",
    "        self.num_tasks = params['num_tasks']\n",
    "\n",
    "        self.train_data = train_data\n",
    "        self.y_train = y_train\n",
    "        self.x_target_unl = x_target_unl\n",
    "        \n",
    "        input_dim = self.train_data.shape[1]\n",
    "        feature_extractor = [nn.Linear(input_dim, params['hidden_units_1']), \n",
    "                             nn.Dropout(p=params['dropout']),\n",
    "                             nn.ReLU()]\n",
    "        self.feature_extractor = nn.Sequential(*feature_extractor)\n",
    "\n",
    "        latent_basis =  [nn.Linear(params['hidden_units_1'], params['latent_space']),\n",
    "                         nn.Dropout(p=params['dropout']),\n",
    "                         nn.ReLU()]\n",
    "        self.latent_basis = nn.Sequential(*latent_basis)\n",
    "        \n",
    "        #task-specific weights\n",
    "        self.S = nn.Linear(params['latent_space'], self.num_tasks)\n",
    "        \n",
    "        #decoder weights\n",
    "        A = [nn.Linear( self.num_tasks , params['latent_space']), nn.ReLU()]\n",
    "        self.A = nn.Sequential(*A)\n",
    "        \n",
    "        #uncertainty (aleatoric)\n",
    "        self.log_vars = torch.zeros(self.num_tasks, requires_grad=True, device=device)\n",
    "        \n",
    "        #discriminator\n",
    "        domain_classifier = [nn.Linear(params[\"latent_space\"], params[\"n_units_disc\"]), \n",
    "                             nn.Dropout(p=params['dropout']), \n",
    "                             nn.ReLU(),\n",
    "                             nn.Linear(params[\"n_units_disc\"], 1), \n",
    "                             nn.Sigmoid()\n",
    "                            ]\n",
    "        \n",
    "        self.domain_classifier = nn.Sequential(*domain_classifier)\n",
    "        \n",
    "    def forward(self, input_data, alpha):\n",
    "        \n",
    "        x = self.feature_extractor(input_data)\n",
    "        h = self.latent_basis(x)\n",
    "        preds = self.S(h)\n",
    "        h_hat = self.A(preds)\n",
    "        reverse_feature = ReverseLayerF.apply(h, alpha)\n",
    "        domain_output = self.domain_classifier(reverse_feature)\n",
    "        return preds, h, h_hat, domain_output\n",
    "\n",
    "    def prepare_data(self):\n",
    "        train_dataset = TensorDataset(torch.FloatTensor(self.train_data),\n",
    "                                      torch.FloatTensor(self.y_train))\n",
    "\n",
    "        \n",
    "        target_unl_dataset = TensorDataset(torch.FloatTensor(self.x_target_unl))\n",
    "        \n",
    "        self.train_dataset = train_dataset\n",
    "        self.target_unl_dataset = target_unl_dataset\n",
    "\n",
    "    def train_dataloader(self):\n",
    "        dataloader1 = DataLoader(self.train_dataset, batch_size=self.batch_size, shuffle=True)\n",
    "        dataloader2 = DataLoader(self.target_unl_dataset, batch_size=self.bs_disc, shuffle=True)\n",
    "        self.len_dataloader = min(len(dataloader1), len(dataloader2))\n",
    "        return zip(dataloader1, cycle(dataloader2))\n",
    "\n",
    "    def test_dataloader(self):\n",
    "        dataloader = DataLoader(self.target_unl_dataset, batch_size=len(self.target_unl_dataset), shuffle=False)\n",
    "        return dataloader\n",
    "\n",
    "    def configure_optimizers(self):\n",
    "        \n",
    "        opt_cla = torch.optim.Adagrad([\n",
    "            {'params': self.feature_extractor.parameters()},\n",
    "            {'params': self.latent_basis.parameters()},\n",
    "            {'params': self.S.parameters()},\n",
    "            {'params': self.A.parameters()},\n",
    "            {'params': self.domain_classifier.parameters()},\n",
    "            {'params': self.log_vars}\n",
    "        ], lr=self.learning_rate)\n",
    "\n",
    "        return opt_cla\n",
    "    \n",
    "    def binary_classification_loss(self, preds, labels):\n",
    "        bin_loss = torch.nn.BCELoss()\n",
    "        return bin_loss(preds, labels)\n",
    "\n",
    "        \n",
    "    def mse_ignore_nan(self, preds, labels):\n",
    "        mse_loss = torch.nn.MSELoss(reduction='none')\n",
    "        per_task_loss = torch.zeros(labels.size(1), device=device)\n",
    "\n",
    "        for k in range(labels.size(1)):\n",
    "            precision = torch.exp(-self.log_vars[k])\n",
    "            diff = mse_loss(preds[~torch.isnan(labels[:,k]), k], labels[~torch.isnan(labels[:,k]), k])\n",
    "            per_task_loss[k] = torch.mean(precision * diff + self.log_vars[k])\n",
    "            \n",
    "            \n",
    "        return torch.mean(per_task_loss[~torch.isnan(per_task_loss)]), per_task_loss  \n",
    "    \n",
    "    def mse_ignore_nan_test(self, preds, labels):\n",
    "        mse_loss = torch.nn.MSELoss(reduction='mean')\n",
    "        per_task_loss = torch.zeros(labels.size(1), device=device)\n",
    "        per_sample_loss = torch.zeros(labels.size(0), device=device)\n",
    "\n",
    "        \n",
    "        for k in range(labels.size(1)):\n",
    "            per_task_loss[k] = mse_loss(preds[~torch.isnan(labels[:,k]), k], labels[~torch.isnan(labels[:,k]), k])\n",
    "        \n",
    "        #per class loss\n",
    "        for k in range(labels.size(0)):\n",
    "            per_sample_loss[k] = mse_loss(preds[k, ~torch.isnan(labels[k,:])], labels[k, ~torch.isnan(labels[k, :])])\n",
    "        \n",
    "        \n",
    "        return torch.mean(per_task_loss[~torch.isnan(per_task_loss)]), per_task_loss,per_sample_loss \n",
    "\n",
    "    def MSE_loss(self, x, x_hat):\n",
    "        mse_loss = torch.nn.MSELoss()\n",
    "        return mse_loss(x, x_hat)\n",
    "    \n",
    "    def forward_pass(self, fw_batch, batch_idx):\n",
    "        x, y = fw_batch[0]\n",
    "        #unlabelled data (target)\n",
    "        unl = fw_batch[1][0]\n",
    "        \n",
    "        #warm-up\n",
    "        p = float(batch_idx +  self.current_epoch * self.len_dataloader) / self.n_epoch / self.len_dataloader\n",
    "        alpha = 2. / (1. + np.exp(-10 * p)) - 1\n",
    "        \n",
    "        preds_simulation = torch.torch.zeros(y.size(0),y.size(1), self.passes, device=device)\n",
    "        for simulation in range(self.passes):\n",
    "            preds, h, h_hat, domain_out_source = self.forward(x, alpha)\n",
    "            preds_simulation[:,:, simulation]=preds\n",
    "\n",
    "        preds_mean = torch.mean(preds_simulation, axis=2)\n",
    "        preds_var = torch.var(preds_simulation, axis=2)\n",
    "        total_unc = torch.mean(preds_var, axis=0)\n",
    "\n",
    "        m_loss, task_loss = self.mse_ignore_nan(preds_mean, y)\n",
    "        recon_loss = self.gamma * self.MSE_loss(h, h_hat)\n",
    "        a = 1 + (total_unc + torch.sum(torch.abs(self.A[0].weight.T),1))\n",
    "        loss_weight = ( a[~torch.isnan(task_loss)] ) * task_loss[~torch.isnan(task_loss)]\n",
    "        loss_weight = torch.sum(loss_weight)\n",
    "        l1_S = self.mu * self.S.weight.norm(1)\n",
    "        L = self.latent_basis[0].weight.norm(2) + self.feature_extractor[0].weight.norm(2)\n",
    "        l2_L = self.lambda_ * L\n",
    "\n",
    "        #domain discriminator source\n",
    "        zeros = torch.zeros(y.size(0), device=self.device)\n",
    "        d_loss_source = self.binary_classification_loss(domain_out_source, zeros)\n",
    "        \n",
    "        #domain discriminator target\n",
    "        preds, h, h_hat, domain_out_target = self.forward(unl, alpha)\n",
    "        ones = torch.ones(unl.size(0), device=self.device)\n",
    "        d_loss_target = self.binary_classification_loss(domain_out_target, ones)\n",
    "        \n",
    "        d_loss = d_loss_source + d_loss_target\n",
    "\n",
    "        #total loss\n",
    "        loss = loss_weight + recon_loss + l1_S + l2_L + (self.lambda_disc *d_loss)\n",
    "        return loss, m_loss, d_loss\n",
    "\n",
    "    def training_step(self, train_batch, batch_idx):\n",
    "        \n",
    "        loss, task_loss, disc_loss = self.forward_pass(train_batch, batch_idx)\n",
    "        logs = {'total_loss': loss, 'source_loss': task_loss, 'disc_loss': disc_loss}\n",
    "        return {'loss': loss, 'log': logs}\n",
    "    \n",
    "    def test_step(self, test_batch, batch_idx):\n",
    "    \n",
    "        x_unl = test_batch[0]\n",
    "\n",
    "        #dropout on\n",
    "        self.feature_extractor[1].train()\n",
    "        self.latent_basis[1].train()\n",
    "        \n",
    "        #TARGET STEPS\n",
    "        #get model preds \n",
    "        preds_simulation = torch.torch.zeros(x_unl.size(0),self.num_tasks, self.passes, device=device)\n",
    "    \n",
    "        for simulation in range(self.passes):\n",
    "            \n",
    "            seed = simulation\n",
    "            random.seed(seed)\n",
    "            np.random.seed(seed)\n",
    "            torch.manual_seed(seed)\n",
    "            torch.backends.cudnn.deterministic = True\n",
    "            torch.backends.cudnn.benchmark = False\n",
    "            \n",
    "            preds, h_target_ls, _, _  = self.forward(x_unl, 0)\n",
    "            preds_simulation[:,:, simulation]=preds\n",
    "            \n",
    "        preds_mean = torch.mean(preds_simulation, axis=2)\n",
    "        \n",
    "        #disable dropouts: train() # resets dropout to train\n",
    "        self.feature_extractor[1].eval()\n",
    "        self.latent_basis[1].eval()\n",
    "        \n",
    "        return {'preds': preds_mean.detach().cpu().numpy()\n",
    "                }\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cell-line dataset;\n",
    "gdsc_dataset = pd.read_csv('data/GDSCDA_fpkm_AUC_all_drugs.zip', index_col=0)\n",
    "#gene set range\n",
    "gene_list = gdsc_dataset.columns[0:1780]\n",
    "drug_list = gdsc_dataset.columns[1780:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pdx novartis dataset;\n",
    "pdx_dataset = pd.read_csv('data/PDX_MTL_DA.csv', index_col=0)\n",
    "drugs_pdx = pdx_dataset.columns[1780:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#best set of hyperparameters found on validation settings;\n",
    "net_params = {\n",
    " 'hidden_units_1': 1500,\n",
    " 'latent_space': 800,\n",
    " 'lr': 0.001,\n",
    " 'dropout': 0.1,\n",
    " 'mu': 1,\n",
    " 'lambda_': 1,\n",
    " 'gamma': 0.01,\n",
    " 'n_units_disc': 500,\n",
    " 'n_epochs': 50,\n",
    " 'bs_disc': 64,\n",
    " 'bs_source': 300,\n",
    " 'lambda_disc': 0.3,\n",
    " 'num_tasks': 200,\n",
    " 'passes': 50}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True, used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "CUDA_VISIBLE_DEVICES: [0]\n",
      "\n",
      "  | Name              | Type       | Params\n",
      "-------------------------------------------------\n",
      "0 | feature_extractor | Sequential | 2 M   \n",
      "1 | latent_basis      | Sequential | 1 M   \n",
      "2 | S                 | Linear     | 160 K \n",
      "3 | A                 | Sequential | 160 K \n",
      "4 | domain_classifier | Sequential | 401 K \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'hidden_units_1': 1500, 'latent_space': 800, 'lr': 0.001, 'dropout': 0.1, 'mu': 1, 'lambda_': 1, 'gamma': 0.01, 'n_units_disc': 500, 'n_epochs': 50, 'bs_disc': 64, 'bs_source': 300, 'lambda_disc': 0.3, 'num_tasks': 200, 'passes': 50}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85032b82a48a49f9b2a401ddff2a3dbf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Training: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving latest checkpoint..\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efecc237cac047c694cb837c16fbbe85",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Testing: 0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------------------------------------\n",
      "DATALOADER:0 TEST RESULTS\n",
      "{'preds': array([[0.8763215 , 0.602008  , 0.8809368 , ..., 0.6359824 , 0.737716  ,\n",
      "        0.7230629 ],\n",
      "       [1.0196947 , 0.7056573 , 0.951037  , ..., 0.7362957 , 0.9649322 ,\n",
      "        0.89155716],\n",
      "       [0.83114064, 0.5705545 , 0.9167524 , ..., 0.7746541 , 0.83101714,\n",
      "        0.90812194],\n",
      "       ...,\n",
      "       [0.8948665 , 0.76307493, 0.9733892 , ..., 0.83099943, 0.8795322 ,\n",
      "        0.93821555],\n",
      "       [0.6325121 , 0.66811043, 0.78564125, ..., 0.6174042 , 0.8663467 ,\n",
      "        0.7862173 ],\n",
      "       [0.85000914, 0.5742485 , 0.90454763, ..., 0.4991126 , 0.85457283,\n",
      "        0.89456856]], dtype=float32)}\n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#training and testing\n",
    "print(net_params)\n",
    "\n",
    "metrics_callback = MetricsCallback()\n",
    "\n",
    "gene_list = gdsc_dataset.columns[0:1780]\n",
    "drug_list = gdsc_dataset.columns[1780:]\n",
    "\n",
    "X_train = gdsc_dataset[gene_list].values\n",
    "y_train = gdsc_dataset[drug_list].values\n",
    "\n",
    "X_test = pdx_dataset[gene_list].values\n",
    "y_test = pdx_dataset[drugs_pdx].values\n",
    "\n",
    "X_train_unl = pdx_dataset[gene_list].values\n",
    "\n",
    "trainer = pl.Trainer(\n",
    "    max_epochs=net_params['n_epochs'],\n",
    "    gpus=1 if torch.cuda.is_available() else None,\n",
    "    callbacks=[metrics_callback],\n",
    "    deterministic=True,\n",
    "    reload_dataloaders_every_epoch=True\n",
    ")\n",
    "\n",
    "seed = 42\n",
    "random.seed(seed)\n",
    "np.random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False\n",
    "\n",
    "seed_everything(seed)\n",
    "model = tugda_da(net_params, X_train, y_train, X_train_unl,\n",
    "              X_train, y_train, y_test)\n",
    "\n",
    "trainer.fit(model)\n",
    "\n",
    "results = trainer.test(model)\n",
    "\n",
    "preds = results[0]['preds']"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "experiments.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "08c2c40ca9a148f8a758002b5e7c2a19": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "14a2a846c9b04d3ea0e6fc11572b9eaa": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": "2",
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "16de1710de804b438a176440fe53178e": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": "2",
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "2fa8b5dbc67843b1ab981de0874f649d": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "488217948d624c87bd5fecd9d0225715": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_dfaf0c326e6445fa820d366958c576b8",
      "placeholder": "​",
      "style": "IPY_MODEL_8ad664c6c9ab4168ba9a16e0b63cf176",
      "value": " 1/1 [00:00&lt;00:00,  5.53it/s]"
     }
    },
    "4dd037a7211b4da393fd515f1c0d4c23": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_81b4c6368e32460285244be291d832e1",
      "placeholder": "​",
      "style": "IPY_MODEL_08c2c40ca9a148f8a758002b5e7c2a19",
      "value": " 22/22 [19:44&lt;00:00, 53.85s/it, loss=0.016, v_num=118]"
     }
    },
    "62cc746940da41a697fe2b962f46c1b4": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": "inline-flex",
      "flex": null,
      "flex_flow": "row wrap",
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": "100%"
     }
    },
    "81b4c6368e32460285244be291d832e1": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8ad664c6c9ab4168ba9a16e0b63cf176": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "af5bd5a68f474afaa488a18253e43d5a": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f194e111ca60412cacdd08e31d6bfdae",
       "IPY_MODEL_488217948d624c87bd5fecd9d0225715"
      ],
      "layout": "IPY_MODEL_e78a51bf585e4f4b9337edc34976bacf"
     }
    },
    "c13f5f528c244afb811a386a13e2ac09": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": "initial"
     }
    },
    "dfaf0c326e6445fa820d366958c576b8": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e16ac871b250481994102f3786b204d3": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_f99550480f0c4a4283e742ff7c05a4d2",
       "IPY_MODEL_4dd037a7211b4da393fd515f1c0d4c23"
      ],
      "layout": "IPY_MODEL_62cc746940da41a697fe2b962f46c1b4"
     }
    },
    "e78a51bf585e4f4b9337edc34976bacf": {
     "model_module": "@jupyter-widgets/base",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": "inline-flex",
      "flex": null,
      "flex_flow": "row wrap",
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": "100%"
     }
    },
    "f194e111ca60412cacdd08e31d6bfdae": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Testing: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_14a2a846c9b04d3ea0e6fc11572b9eaa",
      "max": 1,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_c13f5f528c244afb811a386a13e2ac09",
      "value": 1
     }
    },
    "f99550480f0c4a4283e742ff7c05a4d2": {
     "model_module": "@jupyter-widgets/controls",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "Epoch 99: 100%",
      "description_tooltip": null,
      "layout": "IPY_MODEL_16de1710de804b438a176440fe53178e",
      "max": 22,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_2fa8b5dbc67843b1ab981de0874f649d",
      "value": 22
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}