OtherLLM_LanguageTranslation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import configparser\n",
    "\n",
    "# Read the configuration file\n",
    "config = configparser.ConfigParser()\n",
    "config.read('config.ini')\n",
    "\n",
    "# Retrieve the API key from the configuration file\n",
    "api_key = config.get('openai', 'api_key')\n",
    "\n",
    "# Use the API key in your code\n",
    "openai.api_key = api_key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1: Collecting Story Prompts\n",
    "prompts = [\n",
    "    \"In a small town by the river, there lived a mysterious old man.\",\n",
    "    \"The spaceship landed on an unknown planet, revealing a surprising discovery.\",\n",
    "    \"She opened the door to her childhood home, only to find something unexpected.\",\n",
    "    # Add more prompts as needed\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Step 2: Preprocessing Prompts\n",
    "tokenized_prompts = []\n",
    "for prompt in prompts:\n",
    "    # Tokenize prompt into smaller chunks, e.g., sentences or paragraphs\n",
    "    tokenized_prompt = prompt.split(\". \")\n",
    "    tokenized_prompts.extend(tokenized_prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['In a small town by the river, there lived a mysterious old man.',\n",
       " 'The spaceship landed on an unknown planet, revealing a surprising discovery.',\n",
       " 'She opened the door to her childhood home, only to find something unexpected.']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_prompts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 3: Data Augmentation (if desired)\n",
    "# Implement data augmentation techniques here, e.g., synonym replacement, word swapping, etc.\n",
    "\n",
    "# Step 4: Create Prompt-Response Pairs\n",
    "prompt_response_pairs = []\n",
    "for tokenized_prompt in tokenized_prompts:\n",
    "    # Generate response from the GPT-4 API\n",
    "    response = openai.Completion.create(\n",
    "        model=\"text-davinci-003\",\n",
    "        temperature=0.7,\n",
    "        prompt=tokenized_prompt,\n",
    "        max_tokens=1024,  # Set the desired maximum length of the generated story\n",
    "        n=1,  # Set the number of responses to generate\n",
    "        stop=None,  # Set any stopping criteria, if needed\n",
    "    )\n",
    "    # Extract the generated story from the API response\n",
    "    generated_story = response.choices[0].text.strip()\n",
    "    # Create prompt-response pair\n",
    "    prompt_response_pair = (tokenized_prompt, generated_story)\n",
    "    prompt_response_pairs.append(prompt_response_pair)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('In a small town by the river, there lived a mysterious old man.',\n",
       "  \"He was a quiet man who rarely spoke to anyone, but the townspeople knew he was wise and kind. Each day, the old man would take long walks along the riverbank, collecting sticks and stones that he would later use to build sculptures and small works of art.\\n\\nWhenever someone in the town was in need of advice or help, they would seek out the old man and he would always offer wisdom and kindness. He had a deep understanding of the world and could often offer insight into seemingly impossible problems.\\n\\nOver the years, the old man's reputation grew and soon people from all over the area were coming to him for advice. He was respected and admired by the townspeople, who saw him as a wise and benevolent figure.\\n\\nThe old man's life was quiet and he was content with his simple existence. He never asked for anything in return and would often give away his sculptures and artwork to those in need. He was a mysterious figure who left a lasting impression on the townspeople, who will never forget the kindness and wisdom of the mysterious old man by the river.\"),\n",
       " ('The spaceship landed on an unknown planet, revealing a surprising discovery.',\n",
       "  'The planet was inhabited by an alien species, who were friendly and welcoming to the crew of the spaceship. The crew was amazed to find a thriving and advanced civilization on the planet, far beyond what they had ever imagined. They soon realized that the aliens had developed their own technology and had achieved a level of prosperity and comfort far beyond what was found on earth. After making contact with the aliens, the crew of the spaceship exchanged knowledge and ideas, strengthening the bond between both species. The crew eventually set off to explore the planet and its many wonders, and the aliens continued to foster a strong relationship with them for years to come.'),\n",
       " ('She opened the door to her childhood home, only to find something unexpected.',\n",
       "  'Inside the house was an unfamiliar family, bustling around the kitchen. She could smell the aroma of dinner cooking and hear the children laugh. The woman of the house turned around and saw the girl in the doorway. She smiled warmly and held out her hand. \"Welcome,\" she said, \"You must be the new owners. We were just getting ready to leave. Please, come in and make yourselves at home.\"')]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prompt_response_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prompt Engineering\n",
    "\n",
    "# Step 1: Adding Context to Prompts\n",
    "original_prompt = \"In a small town by the river, there lived a mysterious old man.\"\n",
    "context = \"He was known for his eccentric behavior and strange inventions.\"\n",
    "\n",
    "# Combine the original prompt with additional context\n",
    "augmented_prompt = original_prompt + \" \" + context\n",
    "\n",
    "# Step 2: Modifying Prompt Format\n",
    "question_format_prompt = \"What happens when \" + original_prompt + \"?\"\n",
    "gap_filling_prompt = \"Complete the story: \" + original_prompt + \" ___________.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"He rarely left his house, but when he did, he would often be seen tinkering with some strange contraption or another.\\n\\nPeople in the town whispered about the old man, wondering what he was up to. Some even said they had seen him flying around town on a strange contraption of his own invention. Some kids even dared each other to sneak into the old man's house and see what he was doing, but no one ever did.\\n\\nOne day, a young boy decided to take on the challenge. He snuck into the old man's house and discovered a workshop full of strange, mechanical devices and gadgets. The old man was not there, but the boy was able to piece together what he was up to.\\n\\nThe old man had invented a time machine! He was using it to travel back in time to experience different moments in history. The boy was amazed and filled with wonder. He quickly ran back to town and told everyone what he had discovered.\\n\\nThe old man soon became famous for his time machine and the people of the town began to respect him for his ingenuity. He was seen as a kind of folk hero, and his invention was celebrated throughout the town.\\n\\nThe old man never revealed the details of his time machine, but it's said that he used it to visit many places and times throughout history. To this day, the people of the town still tell stories about the mysterious old man and his amazing time machine.\""
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response = openai.Completion.create(\n",
    "        model=\"text-davinci-003\",\n",
    "        temperature=0.7,\n",
    "        prompt=augmented_prompt,\n",
    "        max_tokens=1024,  # Set the desired maximum length of the generated story\n",
    "        n=1,  # Set the number of responses to generate\n",
    "        stop=None,  # Set any stopping criteria, if needed\n",
    "    )\n",
    "# Extract the generated story from the API response\n",
    "generated_story = response.choices[0].text.strip()\n",
    "generated_story"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Keyword arguments {'source_lang': 'en'} not recognized.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source text (en): Hello, how are you?\n",
      "Translated text (fr): Bonjour, comment ça va ?\n"
     ]
    }
   ],
   "source": [
    "from transformers import MarianMTModel, MarianTokenizer\n",
    "\n",
    "# Load pre-trained model and tokenizer for English to French translation\n",
    "model_name = 'Helsinki-NLP/opus-mt-en-fr'\n",
    "tokenizer = MarianTokenizer.from_pretrained(model_name)\n",
    "model = MarianMTModel.from_pretrained(model_name)\n",
    "\n",
    "def translate_text(text, source_lang, target_lang):\n",
    "    # Tokenize the input text\n",
    "    input_ids = tokenizer.encode(text, return_tensors='pt', source_lang=source_lang)\n",
    "\n",
    "    # Generate translation\n",
    "    translated_ids = model.generate(input_ids=input_ids, \n",
    "                                    decoder_start_token_id=model.config.pad_token_id, \n",
    "                                    num_beams=4, \n",
    "                                    max_length=128,\n",
    "                                    early_stopping=True)\n",
    "    \n",
    "    # Decode the translated tokens\n",
    "    translated_text = tokenizer.decode(translated_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False)\n",
    "    \n",
    "    return translated_text\n",
    "\n",
    "# Example usage\n",
    "source_text = \"Hello, how are you?\"\n",
    "source_language = \"en\"\n",
    "target_language = \"fr\"\n",
    "\n",
    "translated_text = translate_text(source_text, source_language, target_language)\n",
    "\n",
    "print(f\"Source text ({source_language}): {source_text}\")\n",
    "print(f\"Translated text ({target_language}): {translated_text}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Keyword arguments {'source_lang': 'en'} not recognized.\n",
      "Keyword arguments {'source_lang': 'en'} not recognized.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Perplexity: 2.9522607509748147\n",
      "Accuracy: 1.0\n"
     ]
    }
   ],
   "source": [
    "from transformers import MarianMTModel, MarianTokenizer\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "# Load pre-trained model and tokenizer for English to French translation\n",
    "model_name = 'Helsinki-NLP/opus-mt-en-fr'\n",
    "tokenizer = MarianTokenizer.from_pretrained(model_name)\n",
    "model = MarianMTModel.from_pretrained(model_name)\n",
    "\n",
    "def translate_text(text, source_lang, target_lang):\n",
    "    # Tokenize the input text\n",
    "    input_ids = tokenizer.encode(text, return_tensors='pt', source_lang=source_lang)\n",
    "\n",
    "    # Generate translation\n",
    "    translated_ids = model.generate(input_ids=input_ids, \n",
    "                                    decoder_start_token_id=model.config.pad_token_id, \n",
    "                                    num_beams=4, \n",
    "                                    max_length=128,\n",
    "                                    early_stopping=True)\n",
    "    \n",
    "    # Decode the translated tokens\n",
    "    translated_text = tokenizer.decode(translated_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=False)\n",
    "    \n",
    "    return translated_text\n",
    "\n",
    "# Define evaluation dataset\n",
    "eval_dataset = [\n",
    "    {'source_text': \"Hello, how are you?\", 'target_text': \"Bonjour, comment ça va ?\"},\n",
    "    {'source_text': \"I love this place!\", 'target_text': \"J'adore cet endroit !\"},\n",
    "    # Add more evaluation samples as needed\n",
    "]\n",
    "\n",
    "def evaluate_translation(model, dataset):\n",
    "    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "    model = model.to(device)\n",
    "    model.eval()\n",
    "\n",
    "    total_loss = 0.0\n",
    "    total_correct = 0\n",
    "\n",
    "    dataloader = DataLoader(dataset, batch_size=1, shuffle=False)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for batch in dataloader:\n",
    "            source_text = batch['source_text'][0]\n",
    "            target_text = batch['target_text'][0]\n",
    "\n",
    "            # Translate the source text\n",
    "            translated_text = translate_text(source_text, source_language, target_language)\n",
    "\n",
    "            # Compute perplexity\n",
    "            input_ids = tokenizer.encode(target_text, return_tensors='pt').to(device)\n",
    "            target_ids = tokenizer.encode(target_text, return_tensors='pt').to(device)\n",
    "            outputs = model(input_ids=input_ids, labels=target_ids)\n",
    "            loss = outputs.loss.item()\n",
    "            total_loss += loss\n",
    "\n",
    "            # Compute accuracy\n",
    "            predicted_ids = tokenizer.encode(translated_text, return_tensors='pt').to(device)\n",
    "            correct = torch.all(torch.eq(predicted_ids, target_ids)).item()\n",
    "            total_correct += correct\n",
    "\n",
    "    num_samples = len(dataset)\n",
    "    avg_loss = total_loss / num_samples\n",
    "    accuracy = total_correct / num_samples\n",
    "\n",
    "    return avg_loss, accuracy\n",
    "\n",
    "# Example usage\n",
    "source_language = \"en\"\n",
    "target_language = \"fr\"\n",
    "\n",
    "avg_loss, accuracy = evaluate_translation(model,eval_dataset)\n",
    "perplexity = 2 ** avg_loss\n",
    "\n",
    "print(f\"Perplexity: {perplexity}\")\n",
    "print(f\"Accuracy: {accuracy}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The evaluation results show a perplexity of 2.952 and an accuracy of 1.0, which indicates that the translation model performed well on the evaluation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Source Text: Hello, how are you?\n",
      "Target Text: Bonjour, comment ça va ?\n",
      "Translated Text: <pad> Bonjour, comment allez-vous?</s>\n",
      "Normalized Target Text: bonjour, comment ça va ?\n",
      "\n",
      "Source Text: I love this place!\n",
      "Target Text: J'adore cet endroit !\n",
      "Translated Text: <pad> J'adore cet endroit!</s>\n",
      "Normalized Target Text: j'adore cet endroit !\n",
      "\n",
      "Source Text: What time is it?\n",
      "Target Text: Quelle heure est-il ?\n",
      "Translated Text: <pad> Quelle heure est-il?</s>\n",
      "Normalized Target Text: quelle heure est-il ?\n",
      "\n",
      "Source Text: Can you help me?\n",
      "Target Text: Pouvez-vous m'aider ?\n",
      "Translated Text: <pad> Tu peux m'aider?</s>\n",
      "Normalized Target Text: pouvez-vous m'aider ?\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Import the required libraries\n",
    "import torch\n",
    "from transformers import T5Tokenizer, T5ForConditionalGeneration\n",
    "from torch import optim\n",
    "import torch.nn as nn\n",
    "\n",
    "from transformers import MarianMTModel, MarianTokenizer\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "# Load pre-trained model and tokenizer for English to French translation\n",
    "model_name = 'Helsinki-NLP/opus-mt-en-fr'\n",
    "tokenizer = MarianTokenizer.from_pretrained(model_name)\n",
    "model = MarianMTModel.from_pretrained(model_name)\n",
    "\n",
    "# Define the evaluation dataset\n",
    "eval_dataset = [\n",
    "    {'source_text': \"Hello, how are you?\", 'target_text': \"Bonjour, comment ça va ?\"},\n",
    "    {'source_text': \"I love this place!\", 'target_text': \"J'adore cet endroit !\"},\n",
    "    {'source_text': \"What time is it?\", 'target_text': \"Quelle heure est-il ?\"},\n",
    "    {'source_text': \"Can you help me?\", 'target_text': \"Pouvez-vous m'aider ?\"},\n",
    "    # Add more evaluation samples as needed\n",
    "]\n",
    "\n",
    "# Tokenization\n",
    "def tokenize_text(text):\n",
    "    tokenized_text = tokenizer.encode(text, return_tensors='pt')\n",
    "    return tokenized_text\n",
    "\n",
    "# Normalization\n",
    "def normalize_text(text):\n",
    "    normalized_text = text.lower()\n",
    "    return normalized_text\n",
    "\n",
    "def multi_task_learning(model, dataset):\n",
    "    # Define loss functions for each task\n",
    "    task1_criterion = nn.CrossEntropyLoss()\n",
    "    \n",
    "    # Define optimizers for each task\n",
    "    task1_optimizer = optim.Adam(model.parameters(), lr=0.001)\n",
    "    \n",
    "    # Training loop for multi-task learning\n",
    "    for epoch in range(num_epochs):\n",
    "        model.train()\n",
    "        running_loss1 = 0.0\n",
    "\n",
    "        for data in dataset:\n",
    "            source_text, target_text = data['source_text'], data['target_text']\n",
    "            \n",
    "            # Tokenize the source and target text\n",
    "            inputs = tokenizer.encode(source_text, return_tensors='pt')\n",
    "            labels_task1 = tokenizer.encode(target_text, return_tensors='pt')\n",
    "            \n",
    "            # Forward pass and compute losses for each task\n",
    "            outputs = model(\n",
    "                input_ids=inputs[0],\n",
    "                decoder_input_ids=labels_task1.input_ids[:, :-1],  # Exclude the last token\n",
    "                labels=labels_task1.input_ids[:, 1:]  # Exclude the first token\n",
    "            )\n",
    "            loss1 = task1_criterion(outputs.logits, labels_task1.input_ids[:, 1:])  # Exclude the first token\n",
    "            \n",
    "            # Backward pass and update parameters for each task\n",
    "            task1_optimizer.zero_grad()\n",
    "            loss1.backward()\n",
    "            task1_optimizer.step()\n",
    "            \n",
    "            running_loss1 += loss1.item()     \n",
    "        # Print training losses for each epoch\n",
    "        print(f\"Epoch [{epoch+1}/{num_epochs}], Task 1 Loss: {running_loss1 / len(dataset)}\")\n",
    "\n",
    "\n",
    "# Distillation\n",
    "def distillation(large_model, small_model, dataset):\n",
    "    # Define optimizer and loss function for distillation\n",
    "    optimizer = optim.Adam(small_model.parameters(), lr=0.001)\n",
    "    distillation_loss = nn.MSELoss()\n",
    "    \n",
    "    # Training loop for distillation\n",
    "    for epoch in range(num_epochs):\n",
    "        large_model.train()\n",
    "        small_model.train()\n",
    "        running_loss = 0.0\n",
    "        \n",
    "        for data in dataset:\n",
    "            inputs, labels = data['source_text'], data['target_text']\n",
    "            \n",
    "            # Forward pass on the large model and get outputs\n",
    "            outputs_large = large_model(inputs)\n",
    "            \n",
    "            # Forward pass on the small model and get outputs\n",
    "            outputs_small = small_model(inputs)\n",
    "            \n",
    "            # Compute distillation loss\n",
    "            loss = distillation_loss(outputs_small, outputs_large.detach())\n",
    "            \n",
    "            # Backward pass and update parameters\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            running_loss += loss.item()\n",
    "        \n",
    "        # Print training loss for each epoch\n",
    "        print(f\"Epoch [{epoch+1}/{num_epochs}], Loss: {running_loss / len(dataset)}\")\n",
    "\n",
    "# Main program\n",
    "if __name__ == \"__main__\":\n",
    "    # Fine-tuning with advanced techniques\n",
    "    num_epochs = 10\n",
    "\n",
    "    # Perform multi-task learning\n",
    "    multi_task_learning(model, eval_dataset)\n",
    "\n",
    "    # Create a smaller model for distillation\n",
    "    small_model = T5ForConditionalGeneration.from_pretrained('t5-small')\n",
    "\n",
    "    # Perform distillation\n",
    "    distillation(model, small_model, eval_dataset)\n",
    "\n",
    "    # Evaluate on the dataset\n",
    "    for sample in eval_dataset:\n",
    "        source_text = sample['source_text']\n",
    "        target_text = sample['target_text']\n",
    "\n",
    "        # Tokenize the source text\n",
    "        tokenized_source_text = tokenize_text(source_text)\n",
    "\n",
    "        # Generate the translation using the model\n",
    "        outputs = model.generate(input_ids=tokenized_source_text, max_length=50, do_sample=True)\n",
    "        translated_text = tokenizer.decode(outputs[0])\n",
    "\n",
    "        # Normalize the target text\n",
    "        normalized_target_text = normalize_text(target_text)\n",
    "\n",
    "        # Print the results\n",
    "        print(\"Source Text:\", source_text)\n",
    "        print(\"Target Text:\", target_text)\n",
    "        print(\"Translated Text:\", translated_text)\n",
    "        print(\"Normalized Target Text:\", normalized_target_text)\n",
    "        print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.5 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}