chroma_tagging.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e732b22f-2822-4271-81fd-34f1e5f08169",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" \n",
    "the script for the morphological analysis of the longitudinal corpus of spoken Czech child-adult interactions\n",
    "\n",
    "MIT License\n",
    "\n",
    "Copyright (c) 2023 Jakub Sláma\n",
    "\n",
    "Permission is hereby granted, free of charge, to any person obtaining a copy\n",
    "of this software and associated documentation files (the \"Software\"), to deal\n",
    "in the Software without restriction, including without limitation the rights\n",
    "to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n",
    "copies of the Software, and to permit persons to whom the Software is\n",
    "furnished to do so, subject to the following conditions:\n",
    "\n",
    "The above copyright notice and this permission notice shall be included in all\n",
    "copies or substantial portions of the Software.\n",
    "\n",
    "THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
    "IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n",
    "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n",
    "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n",
    "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n",
    "OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n",
    "SOFTWARE.\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fc2da09-5c9d-468d-acf5-c88b4fa6ffb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" \n",
    "fundamantelly, all the functions need to be run, but for analyzing a specific file, you need to run only the last function, \n",
    "file_to_file(), which incorporates all the other functions and from a file creates a new file with morphological tiers added\n",
    "\n",
    "for details, explanations and the like, consult the paper submitted to the the journal Language Resources and Evaluation\n",
    "\n",
    "Chromá, Anna – Sláma, Jakub – Matiasovitsová, Klára – Treichelová, Jolana (submitted): \n",
    "A morphologically annotated longitudinal corpus of spoken Czech child-adult interactions. \n",
    "Language Resources and Evaluation.\n",
    "\n",
    "if you have any questions, feel free to contact me at slama@ujc.cas.cz, but please know that I'm a linguist, not a person \n",
    "who specializes in coding, Python etc., so this script is quite clearly not the optimal way to do its job (and processing \n",
    "files with this script is thus rather slow), even though it thus this job as well as it is supposed to\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e2bc39f0-93bc-4b88-9440-ffadee661229",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "a function for using the MorphoDiTa tagger, see https://ufal.mff.cuni.cz/morphodita\n",
    "the directory needs to be adjusted\n",
    "\n",
    "example of use: tokenize(\"vidím Mařenku\")\n",
    "→ output = a list of Token objects: \n",
    "[Token(word='vidím', lemma='vidět', tag='VB-S---1P-AA---'),\n",
    " Token(word='Mařenku', lemma='Mařenka', tag='NNFS4-----A----')]\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "def tokenize(text):\n",
    "    from corpy.morphodita import Tagger\n",
    "    cs_tagger = Tagger(\"./czech-morfflex-pdt-161115/czech-morfflex-pdt-161115.tagger\")\n",
    "    output = list(cs_tagger.tag(text, convert=\"strip_lemma_id\"))\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "26bc3e3f-b9ce-4753-84cd-6ce86a74cb55",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "this function takes a corpus line in the CHAT (CHILDES) format as the input and transforms it into plain text\n",
    "if the line is not to be tagged (e.g. contains only a hesitation sound), the function returns the value \"NA\" instead\n",
    "\n",
    "example of input: \"*MOT:\ttoho &vybavová vybarvování.\"\n",
    "example of output: 'toho vybarvování .'\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "def transform(input):\n",
    "    import re\n",
    "    if input.startswith(\"@\"):\n",
    "        result = \"NA\"\n",
    "        return result\n",
    "    elif input.startswith(\"%\"):\n",
    "        result = \"NA\"\n",
    "        return result \n",
    "    elif input.startswith(\"*CHI:\"):\n",
    "        result = input.replace(\"*CHI:\t\",\"\")\n",
    "    elif input.startswith(\"*MOT:\"):\n",
    "        result = input.replace(\"*MOT:\t\",\"\")\n",
    "    elif input.startswith(\"*FAT:\"):\n",
    "        result = input.replace(\"*FAT:\t\",\"\")\n",
    "    elif input.startswith(\"*GRA:\"):\n",
    "        result = input.replace(\"*GRA:\t\",\"\")\n",
    "    elif input.startswith(\"*SIS:\"):\n",
    "        result = input.replace(\"*SIS:\t\",\"\")\n",
    "    elif input.startswith(\"*SIT:\"):\n",
    "        result = input.replace(\"*SIT:\t\",\"\")\n",
    "    elif input.startswith(\"*BRO:\"):\n",
    "        result = input.replace(\"*BRO:\t\",\"\")\n",
    "    elif input.startswith(\"*ADU:\"):\n",
    "        result = input.replace(\"*ADU:\t\",\"\")\n",
    "    else:\n",
    "        result = input\n",
    "    \n",
    "    # interjections with underscores\n",
    "    result = result.replace(\"_\", \"\")\n",
    "    \n",
    "    # lengthened vowels\n",
    "    result = result.replace(\"e:\",\"e\").replace(\"a:\",\"a\").replace(\"i:\",\"i\").replace(\"y:\",\"y\").replace(\"o:\",\"o\").replace(\"u:\",\"u\").replace(\"á:\",\"á\").replace(\"é:\",\"é\").replace(\"ě:\",\"ě\").replace(\"í:\",\"í\").replace(\"ý:\",\"ý\").replace(\"ó:\",\"ó\").replace(\"ú:\",\"ú\").replace(\"ů:\",\"ů\").replace(\"r:\",\"r\").replace(\"s:\",\"s\").replace(\"š:\",\"š\")\n",
    "    \n",
    "    # remove \"^\", \"(.)\", \"[*]\"\n",
    "    result = result.replace(\"^\", \"\").replace(\"(.)\",\"\").replace(\"[*]\", \"\")\n",
    "    \n",
    "    # remove \"xxx\", \"yyy\"\n",
    "    result = result.replace(\"xxx\", \"\").replace(\"yyy\", \"\")\n",
    "\n",
    "    # remove \"+<\" from the beginning of lines\n",
    "    result = result.replace(\"+<\", \"\")\n",
    "\n",
    "    # remove all material between \"&\" and \">\", e.g. in \"<v &po> [//] v postýlce\"\n",
    "    result = re.sub(r\"&[aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzžAÁBCČDĎEÉĚFGHIÍJKLMNŇOÓPQRŘSŠTŤUÚŮVWXYÝZŽ]+>\", \">\", result)\n",
    "        \n",
    "    # remove \"<xyz>\" followed by \"[/]\", \"[//]\" or e.g. \"[=! básnička]\"\n",
    "    # e.g. [básnička = poem]: *CHI:\t<máme_tady_xxx_a_pěkný_bububínek_je_tam_jedno_kůzlátko_a_už_nevylezlo> [=! básnička].\n",
    "    result = re.sub(r\"<[ aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzžAÁBCČDĎEÉĚFGHIÍJKLMNŇOÓPQRŘSŠTŤUÚŮVWXYÝZŽ]+> \\[//\\]\", \"\", result)\n",
    "    result = re.sub(r\"<[ aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzžAÁBCČDĎEÉĚFGHIÍJKLMNŇOÓPQRŘSŠTŤUÚŮVWXYÝZŽ]+> \\[/\\]\", \"\", result)\n",
    "    result = re.sub(r\"<[ _aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzžAÁBCČDĎEÉĚFGHIÍJKLMNŇOÓPQRŘSŠTŤUÚŮVWXYÝZŽ]+> \\[=! básnička\\]\", \"\", result)\n",
    "    result = re.sub(r\"<[ _aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzžAÁBCČDĎEÉĚFGHIÍJKLMNŇOÓPQRŘSŠTŤUÚŮVWXYÝZŽ]+> \\[=! písnička\\]\", \"\", result)\n",
    "    result = re.sub(r\"<[ _aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzžAÁBCČDĎEÉĚFGHIÍJKLMNŇOÓPQRŘSŠTŤUÚŮVWXYÝZŽ]+> \\[=! zpěv\\]\", \"\", result)\n",
    "    \n",
    "    # renove all material between \"&=\" and a space, including cases such as \"&=imit:xxx\"\n",
    "    # e.g. \"*CHI:\tjenže ten traktor najednou &=imit:rána.\"\n",
    "    result = re.sub(r\"&=[aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzž:AÁBCČDĎEÉĚFGHIÍJKLMNŇOÓPQRŘSŠTŤUÚŮVWXYÝZŽ]+\", \"\", result)\n",
    "    \n",
    "    # remove all material between \"0\" and a space\n",
    "    result = re.sub(r\"0[aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzžAÁBCČDĎEÉĚFGHIÍJKLMNŇOÓPQRŘSŠTŤUÚŮVWXYÝZŽ]+\", \"\", result)\n",
    "    \n",
    "    # remove all material between \"&\" and a space, e.g. \"*MOT:\ttoho &vybavová vybarvování.\"\n",
    "    result = re.sub(r\"&[aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzžAÁBCČDĎEÉĚFGHIÍJKLMNŇOÓPQRŘSŠTŤUÚŮVWXYÝZŽ]+\", \"\", result)\n",
    "    \n",
    "    # <xyz> [=? xxx]\n",
    "    result = re.sub(\"\\[\\=\\? [ aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzžAÁBCČDĎEÉĚFGHIÍJKLMNŇOÓPQRŘSŠTŤUÚŮVWXYÝZŽ]+\\]\", \"\", result)\n",
    "    \n",
    "    # <xyz> [=! xxx]\n",
    "    result = re.sub(\"\\[\\=\\! [ aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzžAÁBCČDĎEÉĚFGHIÍJKLMNŇOÓPQRŘSŠTŤUÚŮVWXYÝZŽ]+\\]\", \"\", result)\n",
    "    \n",
    "    # remove all the remaining \"<\"s, \"*\"s, \"[?]\"s, and \"[!]\"s, e.g. \"*CHI: chci  <žlutou> [?] kytku.\"\n",
    "    result = result.replace(\"<\", \"\").replace(\">\", \"\").replace(\"[?]\", \"\").replace(\"[!]\", \"\")\n",
    "    \n",
    "    # added: remove quote marks\n",
    "    result = result.replace(\"\\\"\", \"\").replace(\"“\", \"\").replace(\"”\", \"\")\n",
    "    \n",
    "    # remove repetition marking, e.g. [x 2]\n",
    "    # an optional space after the number, because there was a line with \"[x 4 ] .\" at which the script broke down\n",
    "    result = re.sub(r\"\\[x [0123456789]+ ?\\]\", \"\", result)\n",
    "    \n",
    "    # \"přišels [:přišel jsi]\" is to be analyzed as \"přišel jsi\"\n",
    "    result = re.sub(\"[aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzžAÁBCČDĎEÉĚFGHIÍJKLMNŇOÓPQRŘSŠTŤUÚŮVWXYÝZŽ]+ \\[:\", \"\", result)\n",
    "    result = result.replace(\"]\", \"\")\n",
    "    \n",
    "    # token ending in @i, @z:ip, @z:ia, @z:in = to be tagged as an interjection\n",
    "    # bacashooga is a random string not overlapping with any existing Czech words\n",
    "    result = result.replace(\"@i\", \"bacashoogacit\")\n",
    "    result = result.replace(\"@z:ip\", \"bacashoogacit\")\n",
    "    result = result.replace(\"@z:ia\", \"bacashoogacit\")\n",
    "    result = result.replace(\"@z:in\", \"bacashoogacit\")\n",
    "    # token ending in @c, @n = tag is to end with -neo\n",
    "    result = result.replace(\"@c\", \"bacashoogachi\")\n",
    "    result = result.replace(\"@n\", \"bacashoogachi\")\n",
    "    # token ending in @z:c = tag is to end with -ciz\n",
    "    result = result.replace(\"@z:c\", \"bacashoogaciz\")\n",
    "    # the function zpracovat() will later re-tag these appropriately\n",
    "    \n",
    "    # if the result consists only of spaces or punctuation marks, return \"NA\"\n",
    "    alt_result = result\n",
    "    alt_result = re.sub(\"[ .,?!0+…]+\", \"\", alt_result)\n",
    "    if alt_result == \"\":\n",
    "        result = \"NA\"\n",
    "        \n",
    "    # Nee > ne\n",
    "    result = result.replace(\"Nee\",\"ne\").replace(\"nee\",\"ne\")\n",
    "    \n",
    "    # formatting adjustment\n",
    "    result = result.replace(\"?\", \" ?\").replace(\"!\", \" !\").replace(\".\", \" .\").replace(\",\", \" ,\").replace(\"  \", \" \")\n",
    "    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "62cf95b4-1d5a-4f54-b5d6-6145583b1efc",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "input: (tag, word, lemma) provided in the Token object by tokenize()\n",
    "extracts the POS information from the tag and returns the POS value in the MOR format\n",
    "lemma in the input as well, because of the tagging of plural invariable nouns\n",
    "word in the input as well, because of the tagging of proper names\n",
    "\n",
    "example of use: pos(\"NNFS4-----A----\", \"Mařenku\", \"Mařenka\")\n",
    "→ output: 'n:prop'\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "def pos(tag, word, lemma):\n",
    "    result = \"\"\n",
    "    if tag.startswith(\"Z\"):\n",
    "        result = \"Z\"\n",
    "    if tag.startswith(\"X\"):\n",
    "        result = \"x\"\n",
    "    elif tag.startswith(\"N\"):\n",
    "        result = \"n\"\n",
    "        if lemma in [\"dveře\", \"peníze\", \"šaty\", \"šatičky\", \"slipy\", \"brýle\", \"brejle\", \"záda\", \"kamna\", \"kalhotky\", \"nůžky\", \"vrata\", \"korále\", \"dvířka\", \"plavky\", \"kalhoty\", \"hodiny\", \"hodinky\", \"boby\", \"kraťasy\", \"jesličky\", \"narozeniny\"]: # pomnožná\n",
    "            result = \"n:pt\" # plural invariable nouns\n",
    "        if word == word.capitalize(): # proper nouns\n",
    "            result = \"n:prop\"\n",
    "            if lemma in [\"Prčice\"]: # a proper noun that is also a plural invariable noun\n",
    "                result = \"n:prop:pt\"\n",
    "    elif tag.startswith(\"A\"):\n",
    "        result = \"adj\"\n",
    "        if tag.startswith(\"AC\"):\n",
    "            result = \"adj:short\"\n",
    "        elif tag.startswith(\"AU\"):\n",
    "            result = \"adj:poss\"\n",
    "    elif tag.startswith(\"P\"):\n",
    "        result = \"pro\"\n",
    "        if tag.startswith(\"PD\"):\n",
    "            result = \"pro:dem\"\n",
    "        elif tag.startswith(\"PP\") or tag.startswith(\"PH\") or tag.startswith(\"P5\"):\n",
    "            result = \"pro:pers\"\n",
    "        elif tag.startswith(\"P1\") or tag.startswith(\"P9\") or tag.startswith(\"PE\") or tag.startswith(\"PJ\"):\n",
    "            result = \"pro:rel\"\n",
    "        elif tag.startswith(\"PS\"):\n",
    "            result = \"pro:poss\"\n",
    "        elif tag.startswith(\"P8\"): # svůj\n",
    "            result = \"pro:poss\"\n",
    "        elif tag.startswith(\"P4\") or tag.startswith(\"PK\") or tag.startswith(\"PQ\"):\n",
    "            result = \"pro:rel/int\"\n",
    "        elif tag.startswith(\"PW\"):\n",
    "            result = \"pro:neg\"\n",
    "        elif tag.startswith(\"PL\") or tag.startswith(\"PZ\"):\n",
    "            result = \"pro:indef\"\n",
    "        elif tag.startswith(\"P6\") or tag.startswith(\"P7\"): # zvratná se, si...\n",
    "            result = \"pro:refl\"\n",
    "        if lemma == \"svůj\":\n",
    "            result = \"pro:refl:poss\"\n",
    "        if lemma == \"čí\":\n",
    "            result = \"pro:int:poss\"\n",
    "\n",
    "    elif tag.startswith(\"C\"):\n",
    "        result = \"num\"\n",
    "        if tag.startswith(\"Cl\") or tag.startswith(\"Cn\"):\n",
    "            result = \"num:card\"\n",
    "        elif tag.startswith(\"Cr\"):\n",
    "            result = \"num:ord\"\n",
    "        elif tag.startswith(\"Cv\"):\n",
    "            result = \"num:mult\"\n",
    "        elif tag.startswith(\"Ca\"):\n",
    "            result = \"num:indef\"\n",
    "        if word.endswith(\"krát\") and lemma.endswith(\"krát\"):\n",
    "            result = \"num:mult\"\n",
    "        \n",
    "    elif tag.startswith(\"V\"):\n",
    "        result = \"v\"\n",
    "        if lemma in [\"moci\", \"muset\", \"smět\"]: # modal verbs\n",
    "            result = \"v:mod\"\n",
    "        if lemma == \"být\":\n",
    "            result = \"v:aux/cop\"\n",
    "\n",
    "    elif tag.startswith(\"D\"):\n",
    "        result = \"adv\"\n",
    "        if lemma in [\"tu\", \"někam\", \"sem\", \"tamhle\", \"támhle\", \"tuhle\", \"tady\", \"tadyhle\", \"zde\", \"tam\", \"tamhle\", \"onde\", \"odtud\", \"odsud\", \"tudy\", \"tudyhle\", \"tamtudy\", \"teď\", \"teďka\", \"nyní\", \"tehdy\", \"tenkrát\", \"onehdy\", \"poté\", \"pak\", \"nato\", \"tak\", \"takto\", \"takhle\", \"tolik\", \"proto\", \"kde\", \"kdekoli\", \"kdekoliv\", \"kam\", \"kamkoli\", \"kamkoliv\", \"odkud\", \"odkudkoli\", \"odkudkoliv\", \"kudy\", \"kudykoli\", \"kudykoliv\", \"kdy\", \"kdykoli\", \"kdykoliv\", \"dokdy\", \"odkdy\", \"nakdy\", \"jak\", \"kolik\", \"proč\", \"nač\", \"dokud\", \"pokud\", \"nakolik\", \"načež\", \"pročež\", \"začež\", \"někde\", \"kdesi\", \"někdy\", \"kdysi\", \"nějak\", \"jaksi\", \"jakkoli\", \"jakkoliv\", \"koliksi\", \"všude\", \"vždy\"]:\n",
    "            result = \"adv:pro\"\n",
    "        if lemma in [\"nikde\", \"nikam\", \"nikudy\", \"nikdy\", \"nijak\", \"nikterak\", \"odnikud\"]: # zájmenná záporná příslovce\n",
    "            result = \"adv:pro:neg\"\n",
    "    \n",
    "    elif tag.startswith(\"R\"):\n",
    "        result = \"prep\"\n",
    "    elif tag.startswith(\"J^\"):\n",
    "        result = \"conj:coord\"\n",
    "    elif tag.startswith(\"J,\"):\n",
    "        result = \"conj:sub\"\n",
    "    elif tag.startswith(\"J*\"):\n",
    "        result = \"conj:coord\"\n",
    "    elif tag.startswith(\"T\"):\n",
    "        result = \"part\"\n",
    "    elif tag.startswith(\"I\"):\n",
    "        result = \"int\"\n",
    "    \n",
    "    if lemma == \"každý\":\n",
    "        result = \"pro:indef\"\n",
    "        \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2223012e-b646-4ae2-9a9a-0699b81f8354",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" \n",
    "input: (tag, word, lemma) provided in the Token object by tokenize()\n",
    "extracts the morphological information from the tag and returns the morphological tag in the MOR format\n",
    "lemma & word in the input as well, because of the tagging of negation and verbal aspect\n",
    "\n",
    "example of use: transform_tag(\"NNFS4-----A----\", \"Mařenku\", \"Mařenka\")\n",
    "→ output: '4&SG&F'\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "def transform_tag(tag, word, lemma):\n",
    "    mark = \"&\"\n",
    "    result = \"\"\n",
    "    \n",
    "    if tag.startswith(\"V\") and tag[10] == \"N\": # negation\n",
    "        result += \"neg-\"\n",
    "    elif (word.startswith(\"ne\") == True) and (lemma.startswith(\"ne\") == False):\n",
    "        result += \"neg-\"\n",
    "            \n",
    "    if tag.startswith(\"V\"):\n",
    "        vid = \"x_vid\"\n",
    "        if lemma in [\"stát\", \"lézt\", \"letět\", \"lítat\", \"volat\", \"nakupovat\", \"pracovat\", \"psát\", \"znát\", \"sedět\", \"čistit\", \"snídat\", \"zobat\", \"klapat\", \"vypadat\", \"papat\", \"ťukat\", \"kakat\", \"bolet\", \"spinkat\", \"čurat\", \"číst\", \"myslit\", \"plakat\", \"pást\", \"umět\", \"zpívat\", \"stydět\", \"běžet\", \"nést\", \"spát\", \"bydlet\", \"skákat\", \"volat\", \"držet\", \"natahovat\", \"smrdět\", \"zlobit\", \"bát\", \"štěkat\", \"potřebovat\", \"kutálet\", \"pomáhat\", \"ležet\", \"pít\", \"kreslit\", \"říkat\", \"jet\", \"chodit\", \"kousat\", \"začínat\", \"přestávat\", \"moci\", \"koupat\", \"mejt\", \"škrabat\", \"dělávat\", \"táhnout\", \"lízat\", \"stříhat\", \"kupovat\", \"hasit\", \"bláznit\", \"cenit\", \"vyrábět\", \"péci\", \"plašit\", \"vozit\", \"vézt\", \"lisovat\", \"zívat\", \"houpat\", \"čarovat\", \"zvětšovat\", \"koukat\", \"přemisťovat\", \"vědět\", \"líbit\", \"česat\", \"vidět\", \"dávat\", \"fungovat\", \"mít\", \"jít\", \"házet\", \"muset\", \"chybět\", \"dělat\", \"být\", \"chtít\", \"jíst\"]:\n",
    "            vid = \"impf\" # some of the most common imperfective verbs\n",
    "        if lemma in [\"pomoci\", \"zazpívat\", \"rozsvítit\", \"vyhrát\", \"přečíst\", \"povědět\", \"kouknout\", \"podívat\", \"sednout\", \"namočit\", \"skočit\", \"vyčurat\", \"ovázat\", \"sundat\", \"uletět\", \"udělat\", \"zabalit\", \"říci\", \"vyrůst\", \"vyletět\", \"lehnout\", \"rozbít\", \"vzít\", \"rozvázat\", \"spadnout\", \"zabít\", \"upadnout\", \"zůstat\", \"leknout\", \"zvednout\", \"umřít\", \"schovat\", \"sníst\", \"vstát\", \"vzbudit\", \"dostat\", \"začít\", \"pokakat\", \"roztrhat\", \"zalézt\", \"narodit\", \"pustit\", \"odnést\", \"nakoupit\", \"vyndat\", \"zahojit\", \"utrhnout\", \"přinést\", \"zmizet\", \"dát\", \"zkusit\", \"koupit\", \"ukázat\", \"počkat\", \"zavřít\", \"spojit\", \"přijít\", \"přijet\", \"otevřít\"]:\n",
    "            vid = \"pf\" # some of the most common perfective verbs\n",
    "\n",
    "        if tag[3] == \"S\":\n",
    "            cislo = \"SG\"\n",
    "        elif tag[3] == \"P\":\n",
    "            cislo = \"PL\"\n",
    "        else: # the value \"–\": infinitive, auxiliary \"být\" in the conditional form\n",
    "            cislo = \"x_cislo\"\n",
    "\n",
    "        if tag.startswith(\"Vs\"):\n",
    "            rod = \"pas\"\n",
    "        elif tag[11] == \"A\":\n",
    "            rod = \"akt\"\n",
    "        else:\n",
    "            rod = \"x_slovesny_rod\"\n",
    "\n",
    "        if tag[2] != \"-\":\n",
    "            if tag[2] == \"Y\" or tag[2] == \"M\":\n",
    "                jmrod = \"M\"\n",
    "            elif tag[2] == \"N\":\n",
    "                jmrod = \"N\"\n",
    "            else:\n",
    "                jmrod = \"F\"\n",
    "        else:\n",
    "            jmrod = \"x_jmenny_rod\"\n",
    "\n",
    "        if tag.startswith(\"Vf\"): # infinitive\n",
    "            result += \"inf\" + mark + vid\n",
    "        elif tag.startswith(\"Vp\"): # past participle\n",
    "            if word.endswith(\"la\") and jmrod == \"F\": # feminine participles in -la obligatorily singular\n",
    "                cislo = \"SG\"\n",
    "            result += cislo + mark + \"past\" + mark + rod + mark + jmrod + mark + vid\n",
    "        elif tag.startswith(\"Vs\"): # passive participle\n",
    "            result += cislo + mark + \"pas\" + mark + jmrod + mark + vid\n",
    "        \n",
    "        else: # if it is neither an infinitive nor a participle\n",
    "            if tag[7] == \"1\" or tag[7] == \"2\" or tag[7] == \"3\":\n",
    "                result += tag[7]\n",
    "            else: # person not specified: infinitive, transgressive\n",
    "                result += \"x_osoba\"\n",
    "            result += mark + cislo\n",
    "            if tag.startswith(\"Vc\"):\n",
    "                result += mark + \"cond\"\n",
    "                rod = \"akt\"\n",
    "            elif tag.startswith(\"Vi\"):\n",
    "                result += mark + \"imp\"\n",
    "                rod = \"akt\"\n",
    "            else:\n",
    "                result += mark + \"ind\"\n",
    "                if tag[8] == \"P\":\n",
    "                    result += mark + \"pres\"\n",
    "                elif tag[8] == \"F\":\n",
    "                    result += mark + \"futur\" \n",
    "            result += mark + rod\n",
    "            result += mark + vid\n",
    "\n",
    "    elif tag.startswith(\"N\") or tag.startswith(\"A\") or tag.startswith(\"P\") or tag.startswith(\"C\"):\n",
    "        if tag[3] == \"P\":\n",
    "            number = \"PL\"\n",
    "        elif tag[3] == \"S\":\n",
    "            number = \"SG\"\n",
    "        elif tag[3] == \"D\":\n",
    "            number = \"PL\"\n",
    "        else:\n",
    "            number = \"x_cislo\"\n",
    "        if lemma in [\"kdo\", \"co\", \"se\"]:\n",
    "            number = \"SG\"\n",
    "            \n",
    "        if tag[4] != \"X\":\n",
    "            pad = tag[4]\n",
    "        else:\n",
    "            pad = \"x_pad\"\n",
    "        result += pad + mark + number\n",
    "        \n",
    "        if tag.startswith(\"NNM\"):\n",
    "            result += mark + \"MA\"\n",
    "        elif tag.startswith(\"NNI\"):\n",
    "            result += mark + \"MI\"\n",
    "        elif lemma == \"co\":\n",
    "            result += mark + \"N\"\n",
    "        else:        \n",
    "            if tag[2] == \"I\" or tag[2] == \"Y\":\n",
    "                result += mark + \"M\"\n",
    "            elif tag[2] == \"X\":\n",
    "                result += mark + \"x_jmenny_rod\"\n",
    "            else:\n",
    "                if tag[2] != \"-\":\n",
    "                    result += mark + tag[2]\n",
    "    \n",
    "        if tag.startswith(\"Cv\"):\n",
    "            result = \"\"\n",
    "        if word.endswith(\"krát\") and lemma.endswith(\"krát\"):\n",
    "            result = \"\"\n",
    "    \n",
    "    else: \n",
    "        if result == \"neg-\":\n",
    "            result = \"neg\"\n",
    "        else:\n",
    "            result = \"\"\n",
    "\n",
    "    if tag.startswith(\"A\"):\n",
    "        if tag[9] == \"2\":\n",
    "            result = \"CP-\" + result\n",
    "        if tag[9] == \"3\":\n",
    "            result = \"SP-\" + result\n",
    "    \n",
    "    if tag.startswith(\"D\"):\n",
    "        if tag[9] == \"2\":\n",
    "            result = \"CP\"\n",
    "        if tag[9] == \"3\":\n",
    "            result = \"SP\"\n",
    "            \n",
    "    if tag.startswith(\"Cv\"):\n",
    "        result = \"\"\n",
    "    \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "571d3170-105d-47f0-b3ea-2418a2826617",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "this function processes an input text\n",
    "the input text is supposed to be the result of the function transform()\n",
    "the function uses the functions pos() and transform_tag()\n",
    "this function assures that tokens with the placeholders starting with the string \"bacashooga\" are treated as required\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "def zpracovat(text):\n",
    "    if (\"bacashoogachi\" in text) or (\"bacashoogaciz\" in text):\n",
    "        caution = True\n",
    "        chi = []\n",
    "        ciz = []\n",
    "        seznam_slov = []\n",
    "        for word in text.split(\" \"):\n",
    "            if word.endswith(\"bacashoogachi\"):\n",
    "                chi.append(True)\n",
    "                ciz.append(False)\n",
    "                seznam_slov.append(word.replace(\"bacashoogachi\", \"\"))\n",
    "            elif word.endswith(\"bacashoogaciz\"):\n",
    "                ciz.append(True)\n",
    "                chi.append(False)\n",
    "                seznam_slov.append(word.replace(\"bacashoogaciz\", \"\"))\n",
    "            else:\n",
    "                chi.append(False)\n",
    "                ciz.append(False)\n",
    "                seznam_slov.append(word)\n",
    "        text = text.replace(\"bacashoogachi\", \"\").replace(\"bacashoogaciz\", \"\")\n",
    "    else:\n",
    "        caution = False\n",
    "\n",
    "    result = tokenize(text)\n",
    "    word_list = []\n",
    "    pos_list = []\n",
    "    tag_list = []\n",
    "    lemma_list = []\n",
    "    for item in result:\n",
    "        word_list.append(item[0])\n",
    "        lemma_list.append(item[1])\n",
    "        tag_list.append(item[2])\n",
    "        pos_list.append(pos(item[2], item[0], item[1]))\n",
    "    i = 0\n",
    "    result = []\n",
    "    while i < len(word_list):\n",
    "        if pos_list[i] != \"Z\":\n",
    "            new_tag = transform_tag(tag_list[i], word_list[i], lemma_list[i])\n",
    "            if new_tag == \"\":\n",
    "                mark2 = \"\"\n",
    "            else:\n",
    "                mark2 = \"-\"\n",
    "            \n",
    "            if caution == True: \n",
    "                if word_list[i] in seznam_slov:\n",
    "                    index = seznam_slov.index(word_list[i])\n",
    "                    if chi[index] == True:\n",
    "                        new_tag += \"-neo\"\n",
    "                    if ciz[index] == True:\n",
    "                        new_tag += \"-ciz\"\n",
    "            \n",
    "            # lexically specified \"exceptions\": \"mami\" always to be tagged as \"n|máma-5&SG&F\" etc.\n",
    "            if word_list[i] == \"mami\":\n",
    "                result.append(\"n|máma-5&SG&F\")\n",
    "            elif word_list[i] == \"no\":\n",
    "                result.append(\"part|no\")\n",
    "            elif word_list[i] == \"koukej\":\n",
    "                result.append(\"v|koukat-2&SG&imp&akt&impf\")\n",
    "            elif word_list[i] == \"zzz\":\n",
    "                result.append(\"x|zzz\")\n",
    "            \n",
    "            # forms of \"rád\" to be tagged as follows\n",
    "            elif word_list[i] == \"rád\":\n",
    "                result.append(\"adj:short|rád-1&SG&M\")\n",
    "            elif word_list[i] == \"ráda\":\n",
    "                result.append(\"adj:short|rád-1&SG&F\")\n",
    "            elif word_list[i] == \"rádo\":\n",
    "                result.append(\"adj:short|rád-1&SG&N\")\n",
    "            elif word_list[i] == \"rádi\":\n",
    "                result.append(\"adj:short|rád-1&PL&M\")\n",
    "            elif word_list[i] == \"rády\":\n",
    "                result.append(\"adj:short|rád-1&PL&F\")\n",
    "\n",
    "            # reflexive pronouns \"se\" and \"si\" to be tagged as follows\n",
    "            elif word_list[i] == \"se\":\n",
    "                result.append(\"pro:refl|se-4&SG\")\n",
    "            elif word_list[i] == \"si\":\n",
    "                result.append(\"pro:refl|se-3&SG\")\n",
    "                \n",
    "            # the uninflected \"jejichž\" to be tagged as follows\n",
    "            elif word_list[i] == \"jejichž\":\n",
    "                result.append(\"pro:rel:poss|jejichž-x_pad&x_cislo&x_jmenny_rod\")\n",
    "                \n",
    "            # double lemmatization for forms of \"aby.*\" and \"kdyby.*\" + \"ses\", \"sis\", and \"zač\"\n",
    "            elif word_list[i] == \"abych\":\n",
    "                result.append(\"conj:sub_v:aux|aby_být-1&SG&cond&akt&impf\")\n",
    "            elif word_list[i] == \"abys\":\n",
    "                result.append(\"conj:sub_v:aux|aby_být-2&SG&cond&akt&impf\")\n",
    "            elif word_list[i] == \"aby\":\n",
    "                result.append(\"conj:sub_v:aux|aby_být-1&x_cislo&cond&akt&impf\")\n",
    "            elif word_list[i] == \"abychom\":\n",
    "                result.append(\"conj:sub_v:aux|aby_být-1&PL&cond&akt&impf\")\n",
    "            elif word_list[i] == \"abyste\":\n",
    "                result.append(\"conj:sub_v:aux|aby_být-2&PL&cond&akt&impf\")\n",
    "            elif word_list[i] == \"abysme\":\n",
    "                result.append(\"conj:sub_v:aux|aby_být-1&PL&cond&akt&impf\")\n",
    "            elif word_list[i] == \"kdybych\":\n",
    "                result.append(\"conj:sub_v:aux|aby_být-1&SG&cond&akt&impf\")\n",
    "            elif word_list[i] == \"kdybys\":\n",
    "                result.append(\"conj:sub_v:aux|aby_být-2&SG&cond&akt&impf\")\n",
    "            elif word_list[i] == \"kdyby\":\n",
    "                result.append(\"conj:sub_v:aux|aby_být-1&x_cislo&cond&akt&impf\")\n",
    "            elif word_list[i] == \"kdybychom\":\n",
    "                result.append(\"conj:sub_v:aux|aby_být-1&PL&cond&akt&impf\")\n",
    "            elif word_list[i] == \"kdybysme\":\n",
    "                result.append(\"conj:sub_v:aux|aby_být-1&PL&cond&akt&impf\")\n",
    "            elif word_list[i] == \"kdybyste\":\n",
    "                result.append(\"conj:sub_v:aux|aby_být-2&PL&cond&akt&impf\")\n",
    "            elif word_list[i] == \"ses\":\n",
    "                result.append(\"pro:refl_v:aux|se_být-4&SG_2&SG&ind&pres&akt&impf\")\n",
    "            elif word_list[i] == \"sis\":\n",
    "                result.append(\"pro:refl_v:aux|se_být-3&SG_2&SG&ind&pres&akt&impf\")\n",
    "            elif word_list[i] == \"zač\":\n",
    "                result.append(\"prep_pro:int|za_co-4&SG&N\")\n",
    "            \n",
    "            elif word_list[i].endswith(\"bacashoogacit\"):\n",
    "                new_word = word_list[i].replace(\"bacashoogacit\", \"\")\n",
    "                result.append(\"int|\" + new_word)\n",
    "\n",
    "            else: \n",
    "                # lemmatizace my/náš/... ne jako já/můj/...\n",
    "                list_my = [\"my\", \"nás\", \"nám\", \"námi\", \"náma\"]\n",
    "                list_náš = [\"náš\", \"našeho\", \"našemu\", \"našem\", \"naším\", \"naše\", \"naší\", \"našou\", \"naši\", \"našich\", \"našim\", \"našimi\", \"našima\"]\n",
    "                list_vy = [\"vy\", \"vás\", \"vám\", \"vámi\", \"váma\"]\n",
    "                list_váš = [\"váš\", \"vašeho\", \"vašemu\", \"vašem\", \"vaším\", \"vaše\", \"vaší\", \"vašou\", \"vaši\", \"vašich\", \"vašim\", \"vašimi\", \"vašima\"]\n",
    "                list_její = [\"její\", \"jejího\", \"jejímu\", \"jejím\", \"jejích\", \"jejími\", \"jejíma\"]\n",
    "                list_sum = list_my + list_náš + list_vy + list_váš + list_její + [\"jeho\", \"jejich\"]\n",
    "                if word_list[i] not in list_sum:\n",
    "                    result.append(pos_list[i] + \"|\" + lemma_list[i] + mark2 + new_tag)\n",
    "                else:\n",
    "                # plural central pronouns to be lemmatized as e.g. \"my\" or \"náš\" rather than forms of \"já\" or \"můj\"\n",
    "                    if word_list[i] in list_my:\n",
    "                        new_lemma = \"my\"\n",
    "                    if word_list[i] in list_náš:\n",
    "                        new_lemma = \"náš\"\n",
    "                    if word_list[i] in list_vy:\n",
    "                        new_lemma = \"vy\"\n",
    "                    if word_list[i] in list_váš:\n",
    "                        new_lemma = \"váš\"\n",
    "                    if word_list[i] in list_její:\n",
    "                        new_lemma = \"její\"\n",
    "                    if word_list[i] == \"jeho\":\n",
    "                        new_lemma = \"jeho\"\n",
    "                    if word_list[i] == \"jejich\":\n",
    "                        new_lemma = \"jejich\"\n",
    "                    result.append(pos_list[i] + \"|\" + new_lemma + mark2 + new_tag)\n",
    "        if pos_list[i] == \"Z\":\n",
    "            result.append(lemma_list[i])\n",
    "        i += 1 \n",
    "    text = \"%mor:\\t\" + \" \".join(result) + \"\\n\"\n",
    "    \n",
    "    # small formal adjustments\n",
    "    text = text.replace(\", .\", \".\")\n",
    "    text = text.replace(\"\\t, \", \"\\t\")\n",
    "    text = text.replace(\"+ . . .\", \"+…\").replace(\"+ …\", \"+…\").replace(\"+ / .\", \"+/.\")\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "21991fe0-0e6b-4f63-8744-8fd0bd65592c",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "a function for small formal adjustments WITHIN THE MAIN LINE, i.e., this is not a necessary part of the morphological analysis\n",
    "→ adds a space before \".\", \"!\" and \"?\" on the main line\n",
    "→ changes +... to +…\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "def mezera_interpunkce(line):\n",
    "    if line.endswith(\"+...\"):\n",
    "        line = line[:-4] + \"+…\"\n",
    "    elif line.endswith(\"+…\"):\n",
    "        pass\n",
    "    elif line.endswith(\"+/.\"):\n",
    "        pass\n",
    "    elif line.endswith(\".\") and line.endswith(\" .\") == False:\n",
    "        line = line[:-1] + \" .\"\n",
    "    elif line.endswith(\"?\") and line.endswith(\" ?\") == False:\n",
    "        line = line[:-1] + \" ?\"\n",
    "    elif line.endswith(\"!\") and line.endswith(\" !\") == False:\n",
    "        line = line[:-1] + \" !\"\n",
    "    return line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b12d811e-1a3d-4725-b6f3-65a9896554a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" \n",
    "this function takes a file (\"path\" in the input) and creates a new file (\"path_goal\"),\n",
    "which includes the added morphological tiers\n",
    "\n",
    "example of use: file_to_file(\"./test_files/aneta.txt\", \"./test_files/aneta_result.txt\")\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "def file_to_file(path, path_goal):\n",
    "    with open(path, \"r\") as file:\n",
    "        file = file.read()\n",
    "        with open(path_goal, \"a\") as file_goal:\n",
    "            for line in file.split(\"\\n\"):\n",
    "                file_goal.write(mezera_interpunkce(line))\n",
    "                if line!= \"\":\n",
    "                    if transform(line) != \"NA\":\n",
    "                        if transform(line) == \".\" or transform(line) == \"0 .\" or transform(line) == \"nee .\" or transform(line) == \"emem .\" or transform(line) == \"mhm .\" or transform(line) == \"hm .\":\n",
    "                            file_goal.write(\"\\n\")\n",
    "                        else:\n",
    "                            file_goal.write(\"\\n\")\n",
    "                            file_goal.write(zpracovat(transform(line)))\n",
    "                    elif transform(line) == \"NA\":\n",
    "                        file_goal.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6026577e-d844-40fa-82a0-73ff60bfd8a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "to process all corpus files within a folder with the function file_to_file(), the following code was used\n",
    "(the folder here was named \"Sara\" and included all corpus files for the child nicknamed \"Sara\")\n",
    "(all the new files will be found in a new folder, titled \"Sara_tagged\")\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "import nltk\n",
    "\n",
    "path = \"/longitudinal_final/corpus/Sara/\"\n",
    "folder = nltk.corpus.PlaintextCorpusReader(path, r\".*\\.txt\", encoding=\"utf-8\")\n",
    "print(path)\n",
    "for file in folder.fileids():\n",
    "    print(file)\n",
    "    if file.startswith(\".\"):\n",
    "        pass\n",
    "    else:\n",
    "        file_old = path + file\n",
    "        file_new = path + \"Sara_tagged/\" + file\n",
    "        file_to_file(file_old, file_new)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}