7_HingBERTs Ensembled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b09a1eb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "bimport pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import accuracy_score, classification_report, f1_score\n",
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from torch.nn import CrossEntropyLoss\n",
    "from torch.optim import AdamW\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "from transformers import get_linear_schedule_with_warmup\n",
    "\n",
    "# Assuming you have train_df and val_df DataFrames with 'utterance' and 'emotion' columns\n",
    "\n",
    "# Encode labels\n",
    "label_encoder = LabelEncoder()\n",
    "df['emotion_encoded'] = label_encoder.fit_transform(df['emotion'])\n",
    "val_df['emotion_encoded'] = label_encoder.transform(val_df['emotion'])\n",
    "\n",
    "# Load Hinglish BERT tokenizer and model\n",
    "tokenizer_bert = AutoTokenizer.from_pretrained(\"l3cube-pune/hing-bert\")\n",
    "model_bert = AutoModelForSequenceClassification.from_pretrained(\"l3cube-pune/hing-bert\", num_labels=len(label_encoder.classes_))\n",
    "\n",
    "# Load Hinglish RoBERTa tokenizer and model\n",
    "tokenizer_roberta = AutoTokenizer.from_pretrained(\"l3cube-pune/hing-roberta\")\n",
    "model_roberta = AutoModelForSequenceClassification.from_pretrained(\"l3cube-pune/hing-roberta\", num_labels=len(label_encoder.classes_))\n",
    "\n",
    "# Tokenize and prepare data\n",
    "def tokenize_data(data, tokenizer, max_length=128):\n",
    "    tokenized_data = tokenizer(data['utterance'].tolist(), truncation=True, padding='max_length', max_length=max_length, return_tensors='pt')\n",
    "    return tokenized_data\n",
    "\n",
    "train_tokenized_bert = tokenize_data(df, tokenizer_bert)\n",
    "val_tokenized_bert = tokenize_data(val_df, tokenizer_bert)\n",
    "\n",
    "train_tokenized_roberta = tokenize_data(df, tokenizer_roberta)\n",
    "val_tokenized_roberta = tokenize_data(val_df, tokenizer_roberta)\n",
    "\n",
    "# Convert labels to PyTorch tensors\n",
    "y_train = torch.tensor(df['emotion_encoded'].values)\n",
    "y_val = torch.tensor(val_df['emotion_encoded'].values)\n",
    "\n",
    "# Create DataLoader for training and validation\n",
    "train_dataset_bert = TensorDataset(train_tokenized_bert['input_ids'], train_tokenized_bert['attention_mask'], y_train)\n",
    "val_dataset_bert = TensorDataset(val_tokenized_bert['input_ids'], val_tokenized_bert['attention_mask'], y_val)\n",
    "\n",
    "train_dataset_roberta = TensorDataset(train_tokenized_roberta['input_ids'], train_tokenized_roberta['attention_mask'], y_train)\n",
    "val_dataset_roberta = TensorDataset(val_tokenized_roberta['input_ids'], val_tokenized_roberta['attention_mask'], y_val)\n",
    "\n",
    "train_dataloader_bert = DataLoader(train_dataset_bert, batch_size=64, shuffle=True)\n",
    "val_dataloader_bert = DataLoader(val_dataset_bert, batch_size=64, shuffle=False)\n",
    "\n",
    "train_dataloader_roberta = DataLoader(train_dataset_roberta, batch_size=64, shuffle=True)\n",
    "val_dataloader_roberta = DataLoader(val_dataset_roberta, batch_size=64, shuffle=False)\n",
    "\n",
    "# Define training loop for Hinglish BERT\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model_bert.to(device)\n",
    "\n",
    "optimizer_bert = AdamW(model_bert.parameters(), lr=2e-5, weight_decay=0.01)\n",
    "criterion_bert = CrossEntropyLoss()\n",
    "\n",
    "total_steps_bert = len(train_dataloader_bert) * num_epochs\n",
    "scheduler_bert = get_linear_schedule_with_warmup(optimizer_bert, num_warmup_steps=0, num_training_steps=total_steps_bert)\n",
    "\n",
    "# Define training loop for Hinglish RoBERTa\n",
    "model_roberta.to(device)\n",
    "\n",
    "optimizer_roberta = AdamW(model_roberta.parameters(), lr=2e-5, weight_decay=0.01)\n",
    "criterion_roberta = CrossEntropyLoss()\n",
    "\n",
    "total_steps_roberta = len(train_dataloader_roberta) * num_epochs\n",
    "scheduler_roberta = get_linear_schedule_with_warmup(optimizer_roberta, num_warmup_steps=0, num_training_steps=total_steps_roberta)\n",
    "\n",
    "num_epochs = 5\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    # Training loop for Hinglish BERT\n",
    "    model_bert.train()\n",
    "    for batch in train_dataloader_bert:\n",
    "        input_ids, attention_mask, labels = batch\n",
    "        input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n",
    "\n",
    "        optimizer_bert.zero_grad()\n",
    "        outputs = model_bert(input_ids, attention_mask=attention_mask)\n",
    "        logits = outputs.logits\n",
    "        loss = criterion_bert(logits, labels)\n",
    "        loss.backward()\n",
    "\n",
    "        clip_grad_norm_(model_bert.parameters(), max_norm=1.0)\n",
    "        optimizer_bert.step()\n",
    "        scheduler_bert.step()\n",
    "\n",
    "    # Training loop for Hinglish RoBERTa\n",
    "    model_roberta.train()\n",
    "    for batch in train_dataloader_roberta:\n",
    "        input_ids, attention_mask, labels = batch\n",
    "        input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n",
    "\n",
    "        optimizer_roberta.zero_grad()\n",
    "        outputs = model_roberta(input_ids, attention_mask=attention_mask)\n",
    "        logits = outputs.logits\n",
    "        loss = criterion_roberta(logits, labels)\n",
    "        loss.backward()\n",
    "\n",
    "        clip_grad_norm_(model_roberta.parameters(), max_norm=1.0)\n",
    "        optimizer_roberta.step()\n",
    "        scheduler_roberta.step()\n",
    "\n",
    "    # Validation\n",
    "    model_bert.eval()\n",
    "    model_roberta.eval()\n",
    "    val_predictions_bert = []\n",
    "    val_true_labels = []\n",
    "    val_predictions_roberta = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for batch in val_dataloader_bert:\n",
    "            input_ids, attention_mask, labels = batch\n",
    "            input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n",
    "\n",
    "            outputs = model_bert(input_ids, attention_mask=attention_mask)\n",
    "            logits = outputs.logits\n",
    "            predictions = torch.argmax(logits, dim=1).cpu().numpy()\n",
    "\n",
    "            val_predictions_bert.extend(predictions)\n",
    "            val_true_labels.extend(labels.cpu().numpy())\n",
    "\n",
    "        for batch in val_dataloader_roberta:\n",
    "            input_ids, attention_mask, labels = batch\n",
    "            input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n",
    "\n",
    "            outputs = model_roberta(input_ids, attention_mask=attention_mask)\n",
    "            logits = outputs.logits\n",
    "            predictions = torch.argmax(logits, dim=1).cpu().numpy()\n",
    "\n",
    "            val_predictions_roberta.extend(predictions)\n",
    "\n",
    "    # Combine predictions from both models (you can experiment with different combination strategies)\n",
    "    val_predictions_ensemble = [max(pred_bert, pred_roberta) for pred_bert, pred_roberta in zip(val_predictions_bert, val_predictions_roberta)]\n",
    "\n",
    "    val_accuracy = accuracy_score(val_true_labels, val_predictions_ensemble)\n",
    "    val_report = classification_report(val_true_labels, val_predictions_ensemble)\n",
    "    val_weighted_f1 = f1_score(val_true_labels, val_predictions_ensemble, average='weighted')\n",
    "\n",
    "    print(f\"Epoch {epoch + 1}/{num_epochs}\")\n",
    "    print(\"Validation Accuracy:\", val_accuracy)\n",
    "    print(\"Validation Classification Report:\\n\", val_report)\n",
    "    print(\"Validation Weighted F1 Score:\", val_weighted_f1)\n",
    "    print(\"-\" * 50)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}