5_Hing-m-bert.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a60b12a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import accuracy_score, classification_report, f1_score\n",
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "\n",
    "# Assuming you have train_df and val_df DataFrames with 'utterance' and 'emotion' columns\n",
    "\n",
    "# Encode labels\n",
    "label_encoder = LabelEncoder()\n",
    "df['emotion_encoded'] = label_encoder.fit_transform(df['emotion'])\n",
    "val_df['emotion_encoded'] = label_encoder.transform(val_df['emotion'])\n",
    "\n",
    "# Load Hinglish BERT tokenizer and model\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"l3cube-pune/hing-mbert\")\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"l3cube-pune/hing-mbert\", num_labels=len(label_encoder.classes_))\n",
    "\n",
    "# Tokenize and prepare data\n",
    "def tokenize_data(data, tokenizer, max_length=128):\n",
    "    tokenized_data = tokenizer(data['utterance'].tolist(), truncation=True, padding='max_length', max_length=max_length, return_tensors='pt')\n",
    "    return tokenized_data\n",
    "\n",
    "train_tokenized = tokenize_data(df, tokenizer)\n",
    "val_tokenized = tokenize_data(val_df, tokenizer)\n",
    "\n",
    "# Convert labels to PyTorch tensors\n",
    "y_train = torch.tensor(df['emotion_encoded'].values)\n",
    "y_val = torch.tensor(val_df['emotion_encoded'].values)\n",
    "\n",
    "# Create DataLoader for training and validation\n",
    "train_dataset = TensorDataset(train_tokenized['input_ids'], train_tokenized['attention_mask'], y_train)\n",
    "val_dataset = TensorDataset(val_tokenized['input_ids'], val_tokenized['attention_mask'], y_val)\n",
    "\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n",
    "val_dataloader = DataLoader(val_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "# Define training loop\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)\n",
    "\n",
    "optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "num_epochs = 5\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    model.train()\n",
    "    for batch in train_dataloader:\n",
    "        input_ids, attention_mask, labels = batch\n",
    "        input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
    "        loss = outputs.loss\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "    # Validation\n",
    "    model.eval()\n",
    "    val_predictions = []\n",
    "    val_true_labels = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for batch in val_dataloader:\n",
    "            input_ids, attention_mask, labels = batch\n",
    "            input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n",
    "\n",
    "            outputs = model(input_ids, attention_mask=attention_mask)\n",
    "            logits = outputs.logits\n",
    "            predictions = torch.argmax(logits, dim=1).cpu().numpy()\n",
    "\n",
    "            val_predictions.extend(predictions)\n",
    "            val_true_labels.extend(labels.cpu().numpy())\n",
    "\n",
    "    val_accuracy = accuracy_score(val_true_labels, val_predictions)\n",
    "    val_report = classification_report(val_true_labels, val_predictions)\n",
    "    val_weighted_f1 = f1_score(val_true_labels, val_predictions, average='weighted')\n",
    "\n",
    "    print(f\"Epoch {epoch + 1}/{num_epochs}\")\n",
    "    print(\"Validation Accuracy:\", val_accuracy)\n",
    "    print(\"Validation Classification Report:\\n\", val_report)\n",
    "    print(\"Validation Weighted F1 Score:\", val_weighted_f1)\n",
    "    print(\"-\" * 50)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "812a59b5",
   "metadata": {},
   "source": [
    "Hing m Bert with Gradient Clipping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b460b473",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import accuracy_score, classification_report, f1_score\n",
    "import torch\n",
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from torch.nn import CrossEntropyLoss\n",
    "from torch.optim import AdamW\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "\n",
    "# Assuming you have train_df and val_df DataFrames with 'utterance' and 'emotion' columns\n",
    "\n",
    "# Encode labels\n",
    "label_encoder = LabelEncoder()\n",
    "df['emotion_encoded'] = label_encoder.fit_transform(df['emotion'])\n",
    "val_df['emotion_encoded'] = label_encoder.transform(val_df['emotion'])\n",
    "\n",
    "# Load Hinglish BERT tokenizer and model\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"l3cube-pune/hing-mbert\")\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"l3cube-pune/hing-mbert\", num_labels=len(label_encoder.classes_))\n",
    "\n",
    "# Tokenize and prepare data\n",
    "def tokenize_data(data, tokenizer, max_length=128):\n",
    "    tokenized_data = tokenizer(data['utterance'].tolist(), truncation=True, padding='max_length', max_length=max_length, return_tensors='pt')\n",
    "    return tokenized_data\n",
    "\n",
    "train_tokenized = tokenize_data(df, tokenizer)\n",
    "val_tokenized = tokenize_data(val_df, tokenizer)\n",
    "\n",
    "# Convert labels to PyTorch tensors\n",
    "y_train = torch.tensor(df['emotion_encoded'].values)\n",
    "y_val = torch.tensor(val_df['emotion_encoded'].values)\n",
    "\n",
    "# Create DataLoader for training and validation\n",
    "train_dataset = TensorDataset(train_tokenized['input_ids'], train_tokenized['attention_mask'], y_train)\n",
    "val_dataset = TensorDataset(val_tokenized['input_ids'], val_tokenized['attention_mask'], y_val)\n",
    "\n",
    "train_dataloader = DataLoader(train_dataset, batch_size=64, shuffle=True)\n",
    "val_dataloader = DataLoader(val_dataset, batch_size=64, shuffle=False)\n",
    "\n",
    "# Define training loop\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "model.to(device)\n",
    "\n",
    "optimizer = AdamW(model.parameters(), lr=1e-5)\n",
    "criterion = CrossEntropyLoss()\n",
    "\n",
    "num_epochs = 10\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    model.train()\n",
    "    for batch in train_dataloader:\n",
    "        input_ids, attention_mask, labels = batch\n",
    "        input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        outputs = model(input_ids, attention_mask=attention_mask)\n",
    "        logits = outputs.logits\n",
    "        loss = criterion(logits, labels)\n",
    "        loss.backward()\n",
    "\n",
    "        # Implement gradient clipping\n",
    "        clip_grad_norm_(model.parameters(), max_norm=1.0)\n",
    "\n",
    "        optimizer.step()\n",
    "\n",
    "    # Validation\n",
    "    model.eval()\n",
    "    val_predictions = []\n",
    "    val_true_labels = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for batch in val_dataloader:\n",
    "            input_ids, attention_mask, labels = batch\n",
    "            input_ids, attention_mask, labels = input_ids.to(device), attention_mask.to(device), labels.to(device)\n",
    "\n",
    "            outputs = model(input_ids, attention_mask=attention_mask)\n",
    "            logits = outputs.logits\n",
    "            predictions = torch.argmax(logits, dim=1).cpu().numpy()\n",
    "\n",
    "            val_predictions.extend(predictions)\n",
    "            val_true_labels.extend(labels.cpu().numpy())\n",
    "\n",
    "    val_accuracy = accuracy_score(val_true_labels, val_predictions)\n",
    "    val_report = classification_report(val_true_labels, val_predictions)\n",
    "    val_weighted_f1 = f1_score(val_true_labels, val_predictions, average='weighted')\n",
    "\n",
    "    print(f\"Epoch {epoch + 1}/{num_epochs}\")\n",
    "    print(\"Validation Accuracy:\", val_accuracy)\n",
    "    print(\"Validation Classification Report:\\n\", val_report)\n",
    "    print(\"Validation Weighted F1 Score:\", val_weighted_f1)\n",
    "    print(\"-\" * 50)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}