Task_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import numpy as np\n",
    "import os\n",
    "import torch\n",
    "import re\n",
    "from tqdm import tqdm\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer ,LancasterStemmer\n",
    "import string\n",
    "\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from sklearn.metrics import f1_score, accuracy_score, multilabel_confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_path = \"pubmed-rct/PubMed_200k_RCT/\"\n",
    "with open(os.path.join(base_path,\"train.txt\") , \"r\") as f:\n",
    "    train_data = f.readlines()\n",
    "\n",
    "with open(os.path.join(base_path,\"dev.txt\") , \"r\") as f:\n",
    "    dev_data = f.readlines()\n",
    "\n",
    "with open(os.path.join(base_path,\"test.txt\") , \"r\") as f:\n",
    "    test_data = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2211861\n"
     ]
    }
   ],
   "source": [
    "train_data_cleaned = []\n",
    "for i in range(len(train_data)):\n",
    "    if not train_data[i].startswith(\"###\") and not train_data[i].startswith(\"\\n\"):\n",
    "        train_data_cleaned.append(train_data[i])\n",
    "\n",
    "dev_data_cleaned = []\n",
    "for i in range(len(dev_data)):\n",
    "    if not dev_data[i].startswith(\"###\") and not dev_data[i].startswith(\"\\n\"):\n",
    "        dev_data_cleaned.append(dev_data[i])\n",
    "\n",
    "test_data_cleaned = []\n",
    "for i in range(len(test_data)):\n",
    "    if not test_data[i].startswith(\"###\") and not test_data[i].startswith(\"\\n\"):\n",
    "        test_data_cleaned.append(test_data[i])\n",
    "\n",
    "train_data = train_data_cleaned\n",
    "dev_data = dev_data_cleaned\n",
    "test_data = test_data_cleaned\n",
    "\n",
    "print(len(train_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = [ x.split(\"\\t\") for  x in train_data]\n",
    "train_data = np.asarray(train_data,'S')\n",
    "\n",
    "dev_data = [ x.split(\"\\t\") for  x in dev_data]\n",
    "dev_data = np.asarray(dev_data,'S')\n",
    "\n",
    "test_data = [ x.split(\"\\t\") for  x in test_data]\n",
    "test_data = np.asarray(test_data,'S')\n",
    "\n",
    "train_Y , train_X = train_data.T\n",
    "dev_Y , dev_X = dev_data.T\n",
    "test_Y , test_X = test_data.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# max_range = 0\n",
    "# min_range = 10\n",
    "# for x  in train_data:\n",
    "#     if(len(x) != 2):\n",
    "#         print(x)\n",
    "#     max_range = max(max_range , len(x))\n",
    "#     min_range = min(min_range , len(x))\n",
    "\n",
    "# print(min_range, max_range)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The labels in the dataset are : [b'BACKGROUND' b'CONCLUSIONS' b'METHODS' b'OBJECTIVE' b'RESULTS']\n",
      "(2211861, 5)\n"
     ]
    }
   ],
   "source": [
    "print(\"The labels in the dataset are :\",np.unique(train_Y))\n",
    "train_Y = train_Y.reshape([-1,1])\n",
    "ohe = OneHotEncoder()\n",
    "ohe.fit(train_Y)\n",
    "train_Y = ohe.transform(train_Y)\n",
    "print(train_Y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = train_X.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[b'The emergence of HIV as a chronic condition means that people living with HIV are required to take more responsibility for the self-management of their condition , including making physical , emotional and social adjustments .\\n', b'This paper describes the design and evaluation of Positive Outlook , an online program aiming to enhance the self-management skills of gay men living with HIV .\\n']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2211861/2211861 [09:48<00:00, 3760.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['emerg hiv chronic condit mean peopl liv hiv requir tak respons self-management condit includ mak phys emot soc adjust', 'pap describ design evalu posit outlook onlin program aim enh self-management skil gay men liv hiv']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "print(train_X[:2])\n",
    "train_X_split = []\n",
    "lancaster=LancasterStemmer()\n",
    "for i in tqdm(range(len(train_X))):\n",
    "    train_X[i] = re.sub(r'\\d+', '#', train_X[i].decode(\"utf-8\") )\n",
    "    word_tokens = word_tokenize(train_X[i])\n",
    "    filtered_sentence = [lancaster.stem(w.lower()) for w in word_tokens if not w.lower() in stop_words and w not in string.punctuation]\n",
    "    train_X[i] = \" \".join(filtered_sentence)\n",
    "\n",
    "print(train_X[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['emerg hiv chronic condit mean peopl liv hiv requir tak respons self-management condit includ mak phys emot soc adjust', 'pap describ design evalu posit outlook onlin program aim enh self-management skil gay men liv hiv', 'study design random control tri men liv hiv austral assign eith interv group us car control group', 'interv group particip onlin group program posit outlook', 'program bas self-efficacy the us self-management approach enh skil confid abl man psychosoc issu assocy hiv dai lif']\n"
     ]
    }
   ],
   "source": [
    "print(train_X[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2211861, 112026)\n"
     ]
    }
   ],
   "source": [
    "vectorizer = TfidfVectorizer()\n",
    "train_X = vectorizer.fit_transform(train_X)\n",
    "vectorizer.get_feature_names_out()\n",
    "print(train_X.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SparseDataset(Dataset):\n",
    "\n",
    "    def __init__(self, mat_csc, label, device='cpu'):\n",
    "        self.dim = mat_csc.shape\n",
    "        self.device = torch.device(device)\n",
    "        \n",
    "        self.data = mat_csc\n",
    "        self.label = label\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.dim[0]\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        \n",
    "        data = torch.FloatTensor(self.data[idx].toarray()).flatten().to(self.device)\n",
    "        label = torch.FloatTensor(self.label[idx].toarray()).flatten().to(self.device)\n",
    "        # print(data.shape)\n",
    "        return data , label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(train_X.shape[1], 2048),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(2048, 1024),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(1024, 512),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(512, 256),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(256, 32),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(32, 5)#,\n",
    "    # torch.nn.Softmax()\n",
    ")\n",
    "\n",
    "dataset = SparseDataset(train_X, train_Y, \"cuda\")\n",
    "dataloader = DataLoader(dataset=dataset,batch_size=256)\n",
    "\n",
    "model.to(\"cuda\")\n",
    "\n",
    "loss_fn = torch.nn.CrossEntropyLoss(reduction='sum')\n",
    "learning_rate = 1e-3\n",
    "optimizer = torch.optim.RMSprop(model.parameters(), lr=learning_rate)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Epoch 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8641/8641 [26:14<00:00,  5.49it/s, Average loss(last 50 batches) =132]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Epoch 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8641/8641 [26:29<00:00,  5.44it/s, Average loss(last 50 batches) =97.6]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Epoch 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8641/8641 [26:29<00:00,  5.44it/s, Average loss(last 50 batches) =67.2]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Epoch 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8641/8641 [26:38<00:00,  5.41it/s, Average loss(last 50 batches) =43.6]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Epoch 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 8641/8641 [26:40<00:00,  5.40it/s, Average loss(last 50 batches) =31.3]\n"
     ]
    }
   ],
   "source": [
    "for e in range(5):\n",
    "    print(\"Starting Epoch %d\"%e)\n",
    "    pbar = tqdm(dataloader)\n",
    "    ctr = 0\n",
    "    total_loss = 0\n",
    "    for x,y in pbar:\n",
    "        # print(x,y)\n",
    "        y_pred = model(x)\n",
    "\n",
    "        # Compute and print loss.\n",
    "        loss = loss_fn(y_pred,y)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        total_loss = total_loss + loss.item()\n",
    "        if ctr % 49 == 0:\n",
    "            pbar.set_postfix({'Average loss(last 50 batches)': total_loss/50.0})\n",
    "            ctr = 0\n",
    "            total_loss = 0\n",
    "        ctr += 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict, \"first_model.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[b'Adrenergic activation is thought to be an important determinant of outcome in subjects with chronic heart failure ( CHF ) , but baseline or serial changes in adrenergic activity have not been previously investigated in a large patient sample treated with a powerful antiadrenergic agent .\\n', b'Systemic venous norepinephrine was measured at baseline , 3 months , and 12 months in the beta-Blocker Evaluation of Survival Trial ( BEST ) , which compared placebo treatment with the beta-blocker/sympatholytic agent bucindolol .\\n']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28932/28932 [00:07<00:00, 3742.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['adrenerg act thought import determin outcom subject chronic heart fail chf baselin ser chang adrenerg act prevy investig larg paty sampl tre pow antiadrenerg ag', 'system ven norepinephrin meas baselin month month beta-blocker evalu surv tri best comp placebo tre beta-blocker/sympatholytic ag bucindolol']\n"
     ]
    }
   ],
   "source": [
    "dev_X = dev_X.tolist()\n",
    "print(dev_X[:2])\n",
    "dev_X_split = []\n",
    "lancaster=LancasterStemmer()\n",
    "for i in tqdm(range(len(dev_X))):\n",
    "    dev_X[i] = re.sub(r'\\d+', '#', dev_X[i].decode(\"utf-8\") )\n",
    "    word_tokens = word_tokenize(dev_X[i])\n",
    "    filtered_sentence = [lancaster.stem(w.lower()) for w in word_tokens if not w.lower() in stop_words and w not in string.punctuation]\n",
    "    dev_X[i] = \" \".join(filtered_sentence)\n",
    "\n",
    "print(dev_X[:2])\n",
    "\n",
    "dev_X = vectorizer.transform(dev_X)\n",
    "dev_Y = ohe.transform(dev_Y.reshape([-1,1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev_dataset = SparseDataset(dev_X, dev_Y, \"cpu\")\n",
    "dev_dataloader = DataLoader(dataset=dev_dataset,batch_size=256)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 114/114 [01:01<00:00,  1.85it/s]\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "model.to(\"cpu\")\n",
    "for x,y in tqdm(dev_dataloader):\n",
    "    y_pred = model(x)\n",
    "    results.append(y_pred)\n",
    "results = torch.vstack(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_idx = torch.argmax(results, 1, keepdim=True)\n",
    "max_idx = torch.stack([torch.arange(28932),max_idx.flatten()]).T\n",
    "one_hot = torch.FloatTensor(results.shape)\n",
    "one_hot.zero_()\n",
    "for i,j in max_idx:\n",
    "    one_hot[i][j] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.39119415240920735\n",
      "0.41255357389741465\n"
     ]
    }
   ],
   "source": [
    "print(f1_score(dev_Y, one_hot.numpy(), average = \"macro\"))\n",
    "print(accuracy_score(dev_Y, one_hot.numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[22847  3510]\n",
      "  [ 1499  1076]]\n",
      "\n",
      " [[18186  6350]\n",
      "  [ 1355  3041]]\n",
      "\n",
      " [[18890   483]\n",
      "  [ 7207  2352]]\n",
      "\n",
      " [[21338  5169]\n",
      "  [ 1029  1396]]\n",
      "\n",
      " [[17471  1484]\n",
      "  [ 5906  4071]]]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(multilabel_confusion_matrix(dev_Y, one_hot.numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "e38a62193ed3dc8399757708fe4fbcffc4c53d8615cc5ddbe64c2dd235a72117"
  },
  "kernelspec": {
   "display_name": "Python 3.7.11 ('ml_healthcare')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}