8-inspect-and-finalize-notes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
      "env: CUDA_VISIBLE_DEVICES=6\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_DEVICE_ORDER=PCI_BUS_ID\n",
    "%env CUDA_VISIBLE_DEVICES=6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the lang code\n",
    "lang_code = \"UA\"\n",
    "\n",
    "# Define final path\n",
    "final_path = \"/home/tajak/Parlamint-translation/Note-translation/Final-data-CSV/ParlaMint-{}.notes.translated.ana.tsv\".format(lang_code)\n",
    "\n",
    "temp_path = \"/home/tajak/Parlamint-translation/Note-translation/Final-data-CSV/before-sorting/ParlaMint-{}.notes.translated-before-sorting.ana.csv\".format(lang_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2677210/651500763.py:2: FutureWarning: The error_bad_lines argument has been deprecated and will be removed in a future version. Use on_bad_lines in the future.\n",
      "\n",
      "\n",
      "  df = pd.read_csv(final_path, sep=\"\\t\", error_bad_lines=False)\n",
      "Skipping line 622: expected 7 fields, saw 10\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>type</th>\n",
       "      <th>content</th>\n",
       "      <th>xml:lang</th>\n",
       "      <th>length</th>\n",
       "      <th>translation</th>\n",
       "      <th>corpus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gap</td>\n",
       "      <td>inaudible</td>\n",
       "      <td>Не чути</td>\n",
       "      <td>uk</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Unheard</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gap</td>\n",
       "      <td>inaudible</td>\n",
       "      <td>Не Чути</td>\n",
       "      <td>uk</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Don't hear it.</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tag       type  content xml:lang  length     translation corpus\n",
       "0  gap  inaudible  Не чути       uk     2.0         Unheard     UA\n",
       "1  gap  inaudible  Не Чути       uk     2.0  Don't hear it.     UA"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Open the dataframe\n",
    "df = pd.read_csv(final_path, sep=\"\\t\")\n",
    "\n",
    "df.head(2)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyse results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>type</th>\n",
       "      <th>content</th>\n",
       "      <th>xml:lang</th>\n",
       "      <th>length</th>\n",
       "      <th>translation</th>\n",
       "      <th>corpus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>61350</td>\n",
       "      <td>45564</td>\n",
       "      <td>61350</td>\n",
       "      <td>61350</td>\n",
       "      <td>61348.000000</td>\n",
       "      <td>61348</td>\n",
       "      <td>61348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>5</td>\n",
       "      <td>11</td>\n",
       "      <td>61281</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>54793</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>note</td>\n",
       "      <td>time</td>\n",
       "      <td>ЛИТВИН В.М.</td>\n",
       "      <td>uk</td>\n",
       "      <td>NaN</td>\n",
       "      <td>♪ I'll be right</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>59869</td>\n",
       "      <td>38257</td>\n",
       "      <td>3</td>\n",
       "      <td>60758</td>\n",
       "      <td>NaN</td>\n",
       "      <td>920</td>\n",
       "      <td>61348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.568348</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.993362</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>86.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          tag   type      content xml:lang        length      translation  \\\n",
       "count   61350  45564        61350    61350  61348.000000            61348   \n",
       "unique      5     11        61281        3           NaN            54793   \n",
       "top      note   time  ЛИТВИН В.М.       uk           NaN  ♪ I'll be right   \n",
       "freq    59869  38257            3    60758           NaN              920   \n",
       "mean      NaN    NaN          NaN      NaN      2.568348              NaN   \n",
       "std       NaN    NaN          NaN      NaN      2.993362              NaN   \n",
       "min       NaN    NaN          NaN      NaN      1.000000              NaN   \n",
       "25%       NaN    NaN          NaN      NaN      1.000000              NaN   \n",
       "50%       NaN    NaN          NaN      NaN      1.000000              NaN   \n",
       "75%       NaN    NaN          NaN      NaN      3.000000              NaN   \n",
       "max       NaN    NaN          NaN      NaN     86.000000              NaN   \n",
       "\n",
       "       corpus  \n",
       "count   61348  \n",
       "unique      1  \n",
       "top        UA  \n",
       "freq    61348  \n",
       "mean      NaN  \n",
       "std       NaN  \n",
       "min       NaN  \n",
       "25%       NaN  \n",
       "50%       NaN  \n",
       "75%       NaN  \n",
       "max       NaN  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe(include=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "221203"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print total no. of words\n",
    "df.length.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>type</th>\n",
       "      <th>content</th>\n",
       "      <th>xml:lang</th>\n",
       "      <th>length</th>\n",
       "      <th>translation</th>\n",
       "      <th>corpus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gap</td>\n",
       "      <td>inaudible</td>\n",
       "      <td>Не чути</td>\n",
       "      <td>uk</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Unheard</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gap</td>\n",
       "      <td>inaudible</td>\n",
       "      <td>Не Чути</td>\n",
       "      <td>uk</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Don't hear it.</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gap</td>\n",
       "      <td>inaudible</td>\n",
       "      <td>не чути</td>\n",
       "      <td>uk</td>\n",
       "      <td>2.0</td>\n",
       "      <td>do not hear</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gap</td>\n",
       "      <td>inaudible</td>\n",
       "      <td>Не чути)</td>\n",
       "      <td>uk</td>\n",
       "      <td>2.0</td>\n",
       "      <td>Do not hear)</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gap</td>\n",
       "      <td>inaudible</td>\n",
       "      <td>(Не чути</td>\n",
       "      <td>uk</td>\n",
       "      <td>2.0</td>\n",
       "      <td>(Can't hear you.)</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>gap</td>\n",
       "      <td>inaudible</td>\n",
       "      <td>НЕ ЧУТИ</td>\n",
       "      <td>uk</td>\n",
       "      <td>2.0</td>\n",
       "      <td>DO NOT HEAR</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>gap</td>\n",
       "      <td>inaudible</td>\n",
       "      <td>Нерозбірливо</td>\n",
       "      <td>uk</td>\n",
       "      <td>1.0</td>\n",
       "      <td>Inexplicable</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>gap</td>\n",
       "      <td>inaudible</td>\n",
       "      <td>нерозбірливо</td>\n",
       "      <td>uk</td>\n",
       "      <td>1.0</td>\n",
       "      <td>inexplicable</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>incident</td>\n",
       "      <td>action</td>\n",
       "      <td>Хвилина мовчання</td>\n",
       "      <td>uk</td>\n",
       "      <td>2.0</td>\n",
       "      <td>A Minute of Silence</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>incident</td>\n",
       "      <td>action</td>\n",
       "      <td>Лунає Державний Гімн України</td>\n",
       "      <td>uk</td>\n",
       "      <td>4.0</td>\n",
       "      <td>The National Hymn of Ukraine</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>incident</td>\n",
       "      <td>action</td>\n",
       "      <td>Лунає Гімн Державний Гімн України</td>\n",
       "      <td>uk</td>\n",
       "      <td>5.0</td>\n",
       "      <td>The Hymn State Hymn of Ukraine</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>incident</td>\n",
       "      <td>action</td>\n",
       "      <td>Лунає Гімн України</td>\n",
       "      <td>uk</td>\n",
       "      <td>3.0</td>\n",
       "      <td>The Moon of the Hymn of Ukraine</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>incident</td>\n",
       "      <td>action</td>\n",
       "      <td>ХВИЛИНА МОВЧАННЯ.</td>\n",
       "      <td>uk</td>\n",
       "      <td>2.0</td>\n",
       "      <td>A MINUTE OF QUIETNESS.</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>incident</td>\n",
       "      <td>action</td>\n",
       "      <td>Лунає гімн України</td>\n",
       "      <td>uk</td>\n",
       "      <td>3.0</td>\n",
       "      <td>The Anthem of Ukraine</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>incident</td>\n",
       "      <td>action</td>\n",
       "      <td>Депутати співають Державний Гімн України</td>\n",
       "      <td>uk</td>\n",
       "      <td>5.0</td>\n",
       "      <td>The MPs sing the National Hymn of Ukraine</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>incident</td>\n",
       "      <td>action</td>\n",
       "      <td>Лунає державний гімн Республіки Польща</td>\n",
       "      <td>uk</td>\n",
       "      <td>5.0</td>\n",
       "      <td>The National Anthem of the Republic of Poland</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>incident</td>\n",
       "      <td>action</td>\n",
       "      <td>Хвилина Мовчання</td>\n",
       "      <td>uk</td>\n",
       "      <td>2.0</td>\n",
       "      <td>A Minute of Silence</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>incident</td>\n",
       "      <td>action</td>\n",
       "      <td>ЛУНАЄ ГІМН УКРАЇНИ</td>\n",
       "      <td>uk</td>\n",
       "      <td>3.0</td>\n",
       "      <td>THE FUN OF UKRAINE</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>incident</td>\n",
       "      <td>action</td>\n",
       "      <td>Лунає гімн України.</td>\n",
       "      <td>uk</td>\n",
       "      <td>3.0</td>\n",
       "      <td>The anthem of Ukraine echoes.</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>incident</td>\n",
       "      <td>action</td>\n",
       "      <td>У виконанні народних депутатів України лунає Д...</td>\n",
       "      <td>uk</td>\n",
       "      <td>9.0</td>\n",
       "      <td>The National Hymn of Ukraine is on the move in...</td>\n",
       "      <td>UA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         tag       type                                            content  \\\n",
       "0        gap  inaudible                                            Не чути   \n",
       "1        gap  inaudible                                            Не Чути   \n",
       "2        gap  inaudible                                            не чути   \n",
       "3        gap  inaudible                                           Не чути)   \n",
       "4        gap  inaudible                                           (Не чути   \n",
       "5        gap  inaudible                                            НЕ ЧУТИ   \n",
       "6        gap  inaudible                                       Нерозбірливо   \n",
       "7        gap  inaudible                                       нерозбірливо   \n",
       "8   incident     action                                   Хвилина мовчання   \n",
       "9   incident     action                       Лунає Державний Гімн України   \n",
       "10  incident     action                  Лунає Гімн Державний Гімн України   \n",
       "11  incident     action                                 Лунає Гімн України   \n",
       "12  incident     action                                  ХВИЛИНА МОВЧАННЯ.   \n",
       "13  incident     action                                 Лунає гімн України   \n",
       "14  incident     action           Депутати співають Державний Гімн України   \n",
       "15  incident     action             Лунає державний гімн Республіки Польща   \n",
       "16  incident     action                                   Хвилина Мовчання   \n",
       "17  incident     action                                 ЛУНАЄ ГІМН УКРАЇНИ   \n",
       "18  incident     action                                Лунає гімн України.   \n",
       "19  incident     action  У виконанні народних депутатів України лунає Д...   \n",
       "\n",
       "   xml:lang  length                                        translation corpus  \n",
       "0        uk     2.0                                            Unheard     UA  \n",
       "1        uk     2.0                                     Don't hear it.     UA  \n",
       "2        uk     2.0                                        do not hear     UA  \n",
       "3        uk     2.0                                       Do not hear)     UA  \n",
       "4        uk     2.0                                  (Can't hear you.)     UA  \n",
       "5        uk     2.0                                        DO NOT HEAR     UA  \n",
       "6        uk     1.0                                       Inexplicable     UA  \n",
       "7        uk     1.0                                       inexplicable     UA  \n",
       "8        uk     2.0                                A Minute of Silence     UA  \n",
       "9        uk     4.0                       The National Hymn of Ukraine     UA  \n",
       "10       uk     5.0                     The Hymn State Hymn of Ukraine     UA  \n",
       "11       uk     3.0                    The Moon of the Hymn of Ukraine     UA  \n",
       "12       uk     2.0                             A MINUTE OF QUIETNESS.     UA  \n",
       "13       uk     3.0                              The Anthem of Ukraine     UA  \n",
       "14       uk     5.0          The MPs sing the National Hymn of Ukraine     UA  \n",
       "15       uk     5.0      The National Anthem of the Republic of Poland     UA  \n",
       "16       uk     2.0                                A Minute of Silence     UA  \n",
       "17       uk     3.0                                 THE FUN OF UKRAINE     UA  \n",
       "18       uk     3.0                      The anthem of Ukraine echoes.     UA  \n",
       "19       uk     9.0  The National Hymn of Ukraine is on the move in...     UA  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(20)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove foreign language"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(61350, 7)\n",
      "|    |   xml:lang |\n",
      "|:---|-----------:|\n",
      "| uk |      60758 |\n",
      "| ru |        590 |\n",
      "| UA |          2 |\n"
     ]
    }
   ],
   "source": [
    "# Inspect whether there is any additional language\n",
    "print(df.shape)\n",
    "print(df[\"xml:lang\"].value_counts().to_markdown())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>type</th>\n",
       "      <th>content</th>\n",
       "      <th>xml:lang</th>\n",
       "      <th>length</th>\n",
       "      <th>translation</th>\n",
       "      <th>corpus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gap</td>\n",
       "      <td>editorial</td>\n",
       "      <td>Technical problem: content could not be proces...</td>\n",
       "      <td>en</td>\n",
       "      <td>11</td>\n",
       "      <td>Technical problem: content would not be proces...</td>\n",
       "      <td>FI</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tag       type                                            content xml:lang  \\\n",
       "0  gap  editorial  Technical problem: content could not be proces...       en   \n",
       "\n",
       "   length                                        translation corpus  \n",
       "0      11  Technical problem: content would not be proces...     FI  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df[\"xml:lang\"] == \"en\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>type</th>\n",
       "      <th>content</th>\n",
       "      <th>xml:lang</th>\n",
       "      <th>length</th>\n",
       "      <th>translation</th>\n",
       "      <th>corpus</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>8503</td>\n",
       "      <td>5087</td>\n",
       "      <td>8503</td>\n",
       "      <td>8503</td>\n",
       "      <td>8503.000000</td>\n",
       "      <td>8503</td>\n",
       "      <td>8503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>8503</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8485</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>note</td>\n",
       "      <td>relatedDocument</td>\n",
       "      <td>Hallituksen esitys eduskunnalle laeiksi euroop...</td>\n",
       "      <td>fi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Government proposal HE 9/2015 vp</td>\n",
       "      <td>FI</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>5087</td>\n",
       "      <td>5087</td>\n",
       "      <td>1</td>\n",
       "      <td>8503</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>8503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.940492</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.366581</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         tag             type  \\\n",
       "count   8503             5087   \n",
       "unique     2                1   \n",
       "top     note  relatedDocument   \n",
       "freq    5087             5087   \n",
       "mean     NaN              NaN   \n",
       "std      NaN              NaN   \n",
       "min      NaN              NaN   \n",
       "25%      NaN              NaN   \n",
       "50%      NaN              NaN   \n",
       "75%      NaN              NaN   \n",
       "max      NaN              NaN   \n",
       "\n",
       "                                                  content xml:lang  \\\n",
       "count                                                8503     8503   \n",
       "unique                                               8503        1   \n",
       "top     Hallituksen esitys eduskunnalle laeiksi euroop...       fi   \n",
       "freq                                                    1     8503   \n",
       "mean                                                  NaN      NaN   \n",
       "std                                                   NaN      NaN   \n",
       "min                                                   NaN      NaN   \n",
       "25%                                                   NaN      NaN   \n",
       "50%                                                   NaN      NaN   \n",
       "75%                                                   NaN      NaN   \n",
       "max                                                   NaN      NaN   \n",
       "\n",
       "             length                       translation corpus  \n",
       "count   8503.000000                              8503   8503  \n",
       "unique          NaN                              8485      1  \n",
       "top             NaN  Government proposal HE 9/2015 vp     FI  \n",
       "freq            NaN                                 3   8503  \n",
       "mean       6.940492                               NaN    NaN  \n",
       "std        4.366581                               NaN    NaN  \n",
       "min        1.000000                               NaN    NaN  \n",
       "25%        5.000000                               NaN    NaN  \n",
       "50%        5.000000                               NaN    NaN  \n",
       "75%        7.000000                               NaN    NaN  \n",
       "max       51.000000                               NaN    NaN  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove instances that are in non-target language\n",
    "df = df[df[\"xml:lang\"] == \"fi\"]\n",
    "\n",
    "# Inspect the results\n",
    "df.describe(include=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "59015"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.length.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(\"{}\".format(final_path), index=False, sep=\"\\t\", quoting=csv.QUOTE_NONE, quotechar=\"\",  escapechar=\"\\\\\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After saving, inspect the saved file - check whether quotes are still present (that you did not delete them by saving the file too many times.)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "parlamint_env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}