starter_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import necessary libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "# onehot encoding\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Embedding, LSTM, Dense\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load your dataset\n",
    "\n",
    "The Data directory contain SwahiliNewsClassification dataset that will be used in this notebook.\n",
    "<br>Swahili News were collected from different websites that provide news in the Swahili language. The creator of the dataset was able to find some websites that provide news in Swahili only and others in different languages including Swahili.<br>\n",
    "Source: [Swahili News Classification](https://www.kaggle.com/datasets/waalbannyantudre/swahili-news-classification-dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'data/SwahiliNewsClassificationDataset.csv'\n",
    "df = pd.read_csv(data_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>content</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SW4670</td>\n",
       "      <td>Bodi ya Utalii Tanzania (TTB) imesema, itafan...</td>\n",
       "      <td>uchumi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SW30826</td>\n",
       "      <td>PENDO FUNDISHA-MBEYA RAIS Dk. John Magufuri, ...</td>\n",
       "      <td>kitaifa</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                            content category\n",
       "0   SW4670   Bodi ya Utalii Tanzania (TTB) imesema, itafan...   uchumi\n",
       "1  SW30826   PENDO FUNDISHA-MBEYA RAIS Dk. John Magufuri, ...  kitaifa"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Normalize the data\n",
    "\n",
    "This step is to normalize the data by removing the punctuation, numbers, and special characters.<br>\n",
    "*Also to remove the stop words and lemmatize the words finally to convert the words to lower case.*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def normalize_text(text):\n",
    "    # Remove punctuation, numbers, and special characters\n",
    "    text = re.sub(r'[^a-zA-Z\\s]', '', text)\n",
    "    # Convert to lowercase\n",
    "    text = text.lower()\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Normalize the text column\n",
    "df['Text'] = df['content'].apply(normalize_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = df['Text'].values\n",
    "labels = df['category'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(' bodi ya utalii tanzania ttb imesema itafanya misafara ya kutangaza utalii kwenye miji minne nchini china kati ya juni  hadi juni  mwaka huumisafara hiyo itatembelea miji ya beijing juni  shanghai juni  nanjig juni  na changsha juni mwenyekiti wa bodi ttb jaji mstaafu thomas mihayo ameyasema hayo kwenye mkutano na waandishi wa habari jijini dar es salaamtunafanya jitihada kuhakikisha tunavuna watalii wengi zaidi kutoka china hasa tukizingatia umuhimu wa soko la sekta ya utalii nchini amesema jaji mihayonovemba  ttb ilifanya ziara kwenye miji ya beijing shanghai chengdu guangzhou na hong kong kutangaza vivutio vya utalii sanjari kuzitangaza safari za ndege za air tanzaniaziara hiyo inaelezwa kuzaa matunda ikiwa ni pamoja na watalii zaidi ya  kuja nchini mei mwaka huu kutembelea vivutio vya utalii',\n",
       " 'uchumi')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "texts[0], labels[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Convert labels to numerical format\n",
    "\n",
    "This step is to convert the labels to numerical format to be able to use them in the model.\n",
    "\n",
    "final outcome will be a one-hot encoded vector for each label."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\Edgar\\AppData\\Local\\Programs\\Python\\Python311\\Lib\\site-packages\\sklearn\\preprocessing\\_encoders.py:972: FutureWarning: `sparse` was renamed to `sparse_output` in version 1.2 and will be removed in 1.4. `sparse_output` is ignored unless you leave `sparse` to its default value.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# one hot encode the labels\n",
    "encoder = OneHotEncoder(sparse=False)\n",
    "labels = encoder.fit_transform(labels.reshape(-1, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array(['afya', 'burudani', 'kimataifa', 'kitaifa', 'michezo', 'uchumi'],\n",
       "       dtype=object)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder.categories_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['encoder/encoder.joblib']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# # save the encoder\n",
    "import joblib\n",
    "joblib.dump(encoder, 'encoder/encoder.joblib')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split the dataset into training and testing sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_texts, test_texts, train_labels, test_labels = train_test_split(texts, labels, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenize and pad sequences"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this section you will be required to tokenize the words and pad the sequences. try different approaches and see which one gives you the best results.  <br>  <br>\n",
    "You can try the Tokenization strategies notebook for more guidance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_words = 1000  # Adjust based on your dataset size\n",
    "max_len = 200      # Adjust based on the average length of your news articles\n",
    "\n",
    "tokenizer = Tokenizer(num_words=max_words)\n",
    "tokenizer.fit_on_texts(train_texts)\n",
    "\n",
    "train_sequences = tokenizer.texts_to_sequences(train_texts)\n",
    "test_sequences = tokenizer.texts_to_sequences(test_texts)\n",
    "\n",
    "train_data = pad_sequences(train_sequences, maxlen=max_len)\n",
    "test_data = pad_sequences(test_sequences, maxlen=max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save tokenizer\n",
    "import json\n",
    "tokenizer_json = tokenizer.to_json()\n",
    "\n",
    "with open('tokenizer/tf_tokenizer.json', 'w', encoding='utf-8') as f:\n",
    "    json.dump(tokenizer_json, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data[0].shape, train_labels[0].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_dim = 50  # Adjust based on your preferences\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(input_dim=max_words, output_dim=embedding_dim, input_length=max_len))\n",
    "model.add(LSTM(100))\n",
    "model.add(Dense(6, activation='softmax'))\n",
    "\n",
    "\n",
    "# Compile the model\n",
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "epochs = 10\n",
    "\n",
    "history = model.fit(train_data, train_labels, validation_split=0.2, batch_size=batch_size, epochs=epochs)\n",
    "\n",
    "# plot loss and accuracy\n",
    "import matplotlib.pyplot as plt\n",
    "plt.plot(history.history['loss'], label='train')\n",
    "plt.plot(history.history['val_loss'], label='test')\n",
    "plt.legend()\n",
    "plt.show()\n",
    "\n",
    "plt.plot(history.history['accuracy'], label='train')\n",
    "plt.plot(history.history['val_accuracy'], label='test')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluate the model\n",
    "loss, accuracy = model.evaluate(test_data, test_labels, verbose=0)\n",
    "print('Accuracy: %f' % (accuracy*100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the model\n",
    "model.save('models/starter_swahili_news_classification_model.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## inference to the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pre-process for inference\n",
    "def pre_process(tokenizer, max_len, input_text):\n",
    "    input_sequence = tokenizer.texts_to_sequences([input_text])\n",
    "    input_data = pad_sequences(input_sequence, maxlen=max_len)\n",
    "    return input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classify_news(model, tokenizer, encoder, max_len, input_text):\n",
    "    input_data = pre_process(tokenizer, max_len, input_text)\n",
    "    pred = model.predict(input_data)\n",
    "    # for each input sample, the model returns a vector of probabilities\n",
    "    # return all classes with their corresponding probabilities\n",
    "    result_dict = {}\n",
    "\n",
    "    for i, category in enumerate(encoder.categories_[0]):\n",
    "        result_dict[category] = str(round(pred[0][i] * 100, 2))+'%'\n",
    "\n",
    "    highest_prob = max(result_dict, key=result_dict.get)\n",
    "\n",
    "    return (result_dict, highest_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pick a random news headline from df\n",
    "news_ = df.sample(1)\n",
    "\n",
    "news_headline = news_['Text'].values[0]\n",
    "news_category = news_['category'].values[0]\n",
    "\n",
    "result_dict, classified_label = classify_news(model, tokenizer, encoder, max_len, news_headline)\n",
    "\n",
    "print(f'News headline: {news_headline}')\n",
    "print(f'Actual category: {news_category}')\n",
    "print(f'Predicted category: {classified_label}')\n",
    "print(f'Confidence scores: {result_dict}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}