build_greek_fasttext_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "32a5c902",
   "metadata": {},
   "source": [
    "## Import dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9fefcaed",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import codecs\n",
    "import time\n",
    "try:\n",
    "    from collections.abc import Mapping\n",
    "    from gensim.models import FastText\n",
    "except:\n",
    "#     print(\"Depencies not found. Make sure you have installed GenSim.\")\n",
    "    !pip install -I gensim\n",
    "#     !pip install -Iv gensim==3.2.0\n",
    "    from collections.abc import Mapping\n",
    "    from gensim.models import FastText"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2372ef89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Directories found in data folder: ['papyri', 'corpus']\n"
     ]
    }
   ],
   "source": [
    "corpus_directories = [path for path in os.listdir('./data') if not(path.startswith('.'))]\n",
    "print('Directories found in data folder:', corpus_directories)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e00a44f",
   "metadata": {},
   "source": [
    "## Locate corpus data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0423832f",
   "metadata": {},
   "outputs": [],
   "source": [
    "force_lowercase = True\n",
    "use_lemma_disambiguation = False # Some lemmas are indicated with a numeric suffix (e.g., 'ὅτι2')\n",
    "\n",
    "# Change to the directory entered (this is necessary to use the codecs.open() method). \n",
    "# TODO: rewrite this corpus iterator without the codecs module.\n",
    "\n",
    "# if not(os.getcwd().split('/')[-1].endswith('corpus')):\n",
    "#     os.chdir(corpus_directory)\n",
    "\n",
    "# This class streams through the corpus when called.\n",
    "\n",
    "def tokenize(string):\n",
    "    output = string\n",
    "    if use_lemma_disambiguation:\n",
    "        pass\n",
    "    else:\n",
    "        # Filter numeric digits from token\n",
    "        output = ''.join(filter(lambda x: not x.isdigit(), string))\n",
    "    if force_lowercase:\n",
    "        return [token.lower() for token in output.split()]\n",
    "    else:\n",
    "        return output.split()\n",
    "    \n",
    "class MySentences(object):\n",
    "    def __iter__(self):\n",
    "        for corpus_dir in corpus_directories: # the directories where the text files are.\n",
    "            for file in os.listdir(f'./data/{corpus_dir}'): \n",
    "                if file.endswith(\".txt\"):\n",
    "                    for line in codecs.open(f'./data/{corpus_dir}/{file}', 'r+'):\n",
    "                        tokens = tokenize(line)\n",
    "                        if len(tokens) > 0:\n",
    "                            yield tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85c16c57",
   "metadata": {},
   "source": [
    "## Instantiate the corpus streamer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d93a0de1",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = MySentences()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a703d78",
   "metadata": {},
   "source": [
    "## Sanity check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f3d12cab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['κύριος', 'δίδυμος', 'χαίρω']\n",
      "['οἶδα', 'ὅτι', 'ὁ', 'ὁ', 'ὁ', 'ἔτος', 'ἔτι', 'πέντε', 'ὁ', 'ἀπό', 'ὁ', 'λέγω', 'ὅτι', 'εἰ', 'μή', 'οὗτος', 'πέμπω', 'ὅπως', 'ἀπέρχομαι', 'εἰς', 'ὁ', 'καί', 'ποιέω', 'γίγνομαι', 'ἐπεί', 'γάρ', 'ἀσχολέω', 'πέμπω', 'ἐλάτης', 'ἐπεί', 'οὐδείς', 'ἐλαύνω']\n",
      "['ἐπερωτάω', 'ὁμολογέω']\n",
      "['λούκιος', 'εὐσεβής', 'καί', 'μάρκος', 'εὐσεβής', 'καί', 'πούπλιος']\n",
      "['δημητρία', 'ὁ', 'καί', 'ἑρμιόνη', 'ἐμός', 'ὁ', 'υἱός']\n",
      "['παρά', 'ἡρακλείδης']\n"
     ]
    }
   ],
   "source": [
    "# Output should resemble: ['ἐν', 'ὁ', 'πρότερος', 'ὅτι2', 'εὔχομαι', 'νύξ', 'καί', 'ἡμέρα', 'ὁράω', ...etc.\n",
    "count = 0\n",
    "for i in sentences:\n",
    "    count += 1\n",
    "    print(i)\n",
    "    if count > 5:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d2b3de0",
   "metadata": {},
   "source": [
    "## Set hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "25a4c009",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Vector size (too small = underfit; too large = overfit)\n",
    "size_input = 300 \n",
    "\n",
    "# Window size (small = paradigmatic model; large = syntagmatic model)\n",
    "window_input = 5 \n",
    "\n",
    "# Minimum word count for inclusion in network\n",
    "min_count_input = 2 # If a word occurs few times then its vector will not be very high quality\n",
    "\n",
    "# Use skip-gram (False means model will train with CBOW)\n",
    "use_skip_gram = False\n",
    "\n",
    "# Use hierarhical softmax\n",
    "use_hs = True\n",
    "\n",
    "# Use negative sampling (only possible if use_hs is False)\n",
    "use_ns = False\n",
    "\n",
    "# Minimum length of char n-grams model will use for training\n",
    "min_char_ngram_len = 2\n",
    "\n",
    "# Maximum length of char n-grams model will use for training\n",
    "max_char_ngram_len = 5\n",
    "\n",
    "# If max_char_ngram_len < min_char_ngram_len, then character n-grams will not be used in training"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a086553",
   "metadata": {},
   "source": [
    "## Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "682c49fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating model . . . \n",
      "\n",
      "Model initialized in 248.383455991745 seconds.\n",
      "\n",
      "Total number of unique words in corpus: 31903.\n"
     ]
    }
   ],
   "source": [
    "print(\"Generating model . . . \")\n",
    "start = time.time()\n",
    "model = FastText(\n",
    "    sentences, \n",
    "    vector_size=size_input, \n",
    "    window=window_input, \n",
    "    min_count=min_count_input, \n",
    "    sg=(use_skip_gram and 1 or 0), \n",
    "    hs=(use_hs and 1 or 0),\n",
    "    negative=(not(use_hs) and use_ns and 1 or 0),\n",
    "    workers=4,\n",
    "    min_n=min_char_ngram_len,\n",
    "    max_n=max_char_ngram_len\n",
    ")\n",
    "\n",
    "# print(\"Initializing model . . . \")\n",
    "# model = FastText(vector_size=size_input, window=window_input, min_count=min_count_input)\n",
    "# print(\"Building vocabulary . . . \")\n",
    "# model.build_vocab(corpus_iterable=sentences)\n",
    "# print(\"Defining examples . . . \")\n",
    "# total_examples = model.corpus_count\n",
    "# print(\"Training model . . . \")\n",
    "# model.train(corpus_iterable=sentences, total_examples=total_examples, epochs=10)\n",
    "\n",
    "\n",
    "# Create a list of all the unique words in the corpus, in case user wants to query all words.\n",
    "#     words_seen = set() # holds lines already seen\n",
    "#     allWords = []\n",
    "#     for line in sentences:\n",
    "#         for word in line:\n",
    "#             if word not in words_seen: # not a duplicate\n",
    "#                 allWords.append(word)\n",
    "#                 words_seen.add(word)   \n",
    "\n",
    "print(\"\\nModel initialized in {0} seconds.\".format(time.time() - start))\n",
    "print(\"\\nTotal number of unique words in corpus: {0}.\".format(len(model.wv)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7478c819",
   "metadata": {},
   "source": [
    "## Save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "12b26fd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_model_name():\n",
    "    model_name = 'ft_' + '&'.join(corpus_directories)\n",
    "    if use_skip_gram:\n",
    "        model_name += '_skipgram'\n",
    "    else:\n",
    "        model_name += '_cbow'\n",
    "    if use_hs:\n",
    "        model_name += '_hs'\n",
    "    if use_ns:\n",
    "        model_name += '_ns'\n",
    "    model_name += f'_{min_char_ngram_len}_to_{max_char_ngram_len}'\n",
    "    \n",
    "    model_name += f'_size{size_input}_window{window_input}_mincount{min_count_input}'\n",
    "    return model_name\n",
    "\n",
    "model_name = generate_model_name()\n",
    "model.save(f'./models/{model_name}.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3056306",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}