notebooks/ocr_evaluator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluating OCR Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import cv2\n",
    "import time\n",
    "import math\n",
    "from collections import Counter\n",
    "import unidecode\n",
    "from abc import ABC, abstractmethod\n",
    "\n",
    "# Import Widgets\n",
    "from ipywidgets import Button, Text, HBox, VBox\n",
    "from IPython.display import display, clear_output\n",
    "\n",
    "sys.path.append('../src')\n",
    "from ocr import characters\n",
    "from ocr.normalization import word_normalization, letter_normalization\n",
    "# Helpers\n",
    "from ocr.helpers import implt, resize, img_extend\n",
    "from ocr.datahelpers import load_words_data, idx2char\n",
    "from ocr.tfhelpers import Model\n",
    "from ocr.viz import print_progress_bar"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Global Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ONLY 'en' is supported right now\n",
    "LANG = 'en'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Trained Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "charClass_1 = Model('../models/char-clas/' + LANG + '/CharClassifier')\n",
    "\n",
    "wordClass = Model('../models/word-clas/' + LANG + '/WordClassifier2', 'prediction_infer')\n",
    "wordClass2 = Model('../models/word-clas/' + LANG + '/SeqRNN/Classifier', 'word_prediction') # None\n",
    "wordClass3 = Model('../models/word-clas/' + LANG + '/CTC/Classifier2', 'word_prediction')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading words...\n",
      "-> Number of words: 267\n",
      " |████████████████████████████████████████| 100.0% \n",
      "\n",
      "Number of chars: 1356\n"
     ]
    }
   ],
   "source": [
    "images, labels = load_words_data('../data/sets/test.csv', is_csv=True)\n",
    "\n",
    "\n",
    "for i in range(len(images)):\n",
    "    print_progress_bar(i, len(images))\n",
    "    images[i] = word_normalization(\n",
    "        cv2.cvtColor(images[i], cv2.COLOR_GRAY2RGB),\n",
    "        60,\n",
    "        border=False,\n",
    "        tilt=True,\n",
    "        hystNorm=True)\n",
    "\n",
    "if LANG == 'en':\n",
    "    for i in range(len(labels)):\n",
    "        labels[i] = unidecode.unidecode(labels[i])\n",
    "print()        \n",
    "print('Number of chars:', sum(len(l) for l in labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Words\n",
    "WORDS = {}\n",
    "with open('../data/dictionaries' + LANG + '_50k.txt') as f:\n",
    "    for line in f:\n",
    "        if LANG == 'en':\n",
    "            WORDS[unidecode.unidecode(line.split(\" \")[0])] = int(line.split(\" \")[1])\n",
    "        else:\n",
    "            WORDS[line.split(\" \")[0]] = int(line.split(\" \")[1])\n",
    "WORDS = Counter(WORDS)\n",
    "\n",
    "def P(word, N=sum(WORDS.values())): \n",
    "    \"Probability of word.\"\n",
    "    return WORDS[word] / N\n",
    "\n",
    "def correction(word): \n",
    "    \"Most probable spelling correction for word.\"\n",
    "    if word in WORDS:\n",
    "        return word\n",
    "    return max(candidates(word), key=P)\n",
    "\n",
    "def candidates(word): \n",
    "    \"Generate possible spelling corrections for word.\"\n",
    "    return (known([word]) or known(edits1(word)) or known(edits2(word)) or [word])\n",
    "\n",
    "def known(words): \n",
    "    \"The subset of words that appear in the dictionary of WORDS.\"\n",
    "    return set(w for w in words if w in WORDS)\n",
    "\n",
    "def edits1(word):\n",
    "    \"All edits that are one edit away from `word`.\"\n",
    "    \n",
    "    if LANG == 'cz':\n",
    "        letters = 'aábcčdďeéěfghiíjklmnňoópqrřsštťuúůvwxyýzž'\n",
    "    else:\n",
    "        letters = 'abcdefghijklmnopqrstuvwxyz'\n",
    "    splits     = [(word[:i], word[i:])    for i in range(len(word) + 1)]\n",
    "    deletes    = [L + R[1:]               for L, R in splits if R]\n",
    "    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]\n",
    "    replaces   = [L + c + R[1:]           for L, R in splits if R for c in letters]\n",
    "    inserts    = [L + c + R               for L, R in splits for c in letters]\n",
    "    return set(deletes + transposes + replaces + inserts)\n",
    "\n",
    "def edits2(word): \n",
    "    \"All edits that are two edits away from `word`.\"\n",
    "    return (e2 for e1 in edits1(word) for e2 in edits1(e1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cer(r, h):\n",
    "    \"\"\"\n",
    "    From two strings calculate character error rate (insert, delete or substitution).\n",
    "    \"\"\"\n",
    "    r = list(r)\n",
    "    h = list(h)\n",
    "    d = np.zeros((len(r) + 1) * (len(h) + 1), dtype=np.uint16)\n",
    "    d = d.reshape((len(r) + 1, len(h) + 1))\n",
    "    for i in range(len(r) + 1):\n",
    "        for j in range(len(h) + 1):\n",
    "            if i == 0:\n",
    "                d[0][j] = j\n",
    "            elif j == 0:\n",
    "                d[i][0] = i\n",
    "\n",
    "    for i in range(1, len(r) + 1):\n",
    "        for j in range(1, len(h) + 1):\n",
    "            if r[i - 1] == h[j - 1]:\n",
    "                d[i][j] = d[i - 1][j - 1]\n",
    "            else:\n",
    "                substitution = d[i - 1][j - 1] + 1\n",
    "                insertion = d[i][j - 1] + 1\n",
    "                deletion = d[i - 1][j] + 1\n",
    "                d[i][j] = min(substitution, insertion, deletion)\n",
    "#     result = float(d[len(r)][len(h)]) / len(r) * 100\n",
    "#     print('CER %.4f %%' % result)\n",
    "#     print(d[len(r)][len(h)])\n",
    "    return(d[len(r)][len(h)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cycler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Cycler(ABC):\n",
    "    \"\"\" Abstract cycler class \"\"\" \n",
    "    def __init__(self,\n",
    "                 images,\n",
    "                 labels,\n",
    "                 charClass,\n",
    "                 stats=\"No Stats Provided\",\n",
    "                 slider=(60, 15),\n",
    "                 ctc=False,\n",
    "                 seq2seq=False,\n",
    "                 charRNN=False):\n",
    "        self.images = images\n",
    "        self.labels = labels\n",
    "        self.charClass = charClass\n",
    "        self.slider = slider\n",
    "        self.totalChars = sum([len(l) for l in labels])\n",
    "        self.ctc = ctc\n",
    "        self.seq2seq = seq2seq\n",
    "        self.charRNN = charRNN\n",
    "        self.stats = stats\n",
    "        \n",
    "        self.evaluate()\n",
    "        \n",
    "    @abstractmethod\n",
    "    def recogniseWord(self, img):\n",
    "        pass\n",
    "    \n",
    "    def countCorrect(self, pred, label, lower=False):\n",
    "        correct = 0\n",
    "        for i in range(min(len(pred), len(label))):\n",
    "            if ((not lower and pred[i] == label[i])\n",
    "                 or (lower and pred[i] == label.lower()[i])):\n",
    "                correct += 1\n",
    "                \n",
    "        return correct        \n",
    "\n",
    "    \n",
    "    def evaluate(self):\n",
    "        \"\"\" Evaluate accuracy of the word classification \"\"\"\n",
    "        print()\n",
    "        print(\"STATS:\", self.stats)\n",
    "        print(self.labels[1], ':', self.recogniseWord(self.images[1]))\n",
    "        start_time = time.time()\n",
    "        for i in range(len(self.images)):\n",
    "            word = self.recogniseWord(self.images[i])\n",
    "#             a = correction(word.lower()\n",
    "        print(\"--- %s seconds ---\" % round(time.time() - start_time, 2))\n",
    "        ccer = 0\n",
    "        correctLetters = 0\n",
    "        correctWords = 0\n",
    "        correctWordsCorrection = 0\n",
    "        correctLettersCorrection = 0\n",
    "        for i in range(len(self.images)):\n",
    "            word = self.recogniseWord(self.images[i])\n",
    "            correctLetters += self.countCorrect(word,\n",
    "                                         self.labels[i])\n",
    "            # Correction works only for lower letters\n",
    "            correctLettersCorrection += self.countCorrect(correction(word.lower()),\n",
    "                                                       self.labels[i],\n",
    "                                                       lower=True)\n",
    "            ccer += cer(word, self.labels[i])\n",
    "            # Words accuracy\n",
    "            if word == self.labels[i]:\n",
    "                correctWords += 1\n",
    "            if correction(word.lower()) == self.labels[i].lower():\n",
    "                correctWordsCorrection += 1\n",
    "\n",
    "        print(\"Correct/Total: %s / %s\" % (correctLetters, self.totalChars))\n",
    "        print(\"CERacc: %s %%\" % round(100 - ccer/self.totalChars * 100, 4))\n",
    "        print(\"Letter Accuracy: %s %%\" % round(correctLetters/self.totalChars * 100, 4))\n",
    "        print(\"Letter Accuracy with Correction: %s %%\" % round(correctLettersCorrection/self.totalChars * 100, 4))\n",
    "        print(\"Word Accuracy: %s %%\" % round(correctWords/len(self.images) * 100, 4))\n",
    "        print(\"Word Accuracy with Correction: %s %%\" % round(correctWordsCorrection/len(self.images) * 100, 4))\n",
    "#         print(\"--- %s seconds ---\" % round(time.time() - start_time, 2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WordCycler(Cycler):\n",
    "    \"\"\" Cycle through the words and recognise them \"\"\" \n",
    "    def recogniseWord(self, img):\n",
    "        slider = self.slider\n",
    "        \n",
    "        if self.ctc:\n",
    "            step = 10    # 10 for (60, 60) slider\n",
    "            img = cv2.copyMakeBorder(\n",
    "                img,\n",
    "                0, 0, self.slider[1]//2, self.slider[1]//2,\n",
    "                cv2.BORDER_CONSTANT,\n",
    "                value=[0, 0, 0])\n",
    "            img = img_extend(\n",
    "                img,\n",
    "                (img.shape[0], max(-(-img.shape[1] // step) * step, self.slider[1] + step)))\n",
    "            length = (img.shape[1]-slider[1]) // step\n",
    "            input_seq = np.zeros((1, length, slider[0] * slider[1]), dtype=np.float32)\n",
    "            input_seq[0][:] = [img[:, loc*step: loc*step + slider[1]].flatten()\n",
    "                             for loc in range(length)]\n",
    "            input_seq = input_seq.swapaxes(0, 1)\n",
    "            \n",
    "            pred = self.charClass.eval_feed({'inputs:0': input_seq,\n",
    "                                             'inputs_length:0': [length],\n",
    "                                             'keep_prob:0': 1})[0]\n",
    "            \n",
    "            word = ''\n",
    "            for i in pred:\n",
    "                if word == 0 and i != 0:\n",
    "                    break\n",
    "                else:\n",
    "                    word += idx2char(i)\n",
    "            \n",
    "        else:       \n",
    "            length = img.shape[1]//slider[1]\n",
    "\n",
    "            input_seq = np.zeros((1, length, slider[0] * slider[1]), dtype=np.float32)\n",
    "            input_seq[0][:] = [img[:, loc * slider[1]: (loc+1) * slider[1]].flatten()\n",
    "                               for loc in range(length)]                                \n",
    "            input_seq = input_seq.swapaxes(0, 1)\n",
    "\n",
    "\n",
    "            if self.seq2seq:\n",
    "                targets = np.zeros((1, 1), dtype=np.int32)  \n",
    "                pred = self.charClass.eval_feed({'encoder_inputs:0': input_seq,\n",
    "                                                 'encoder_inputs_length:0': [length],\n",
    "                                                 'decoder_targets:0': targets,\n",
    "                                                 'keep_prob:0': 1})[0]\n",
    "            else:\n",
    "                targets = np.zeros((1, 1, 4096), dtype=np.int32)  \n",
    "                pred = self.charClass.eval_feed({'encoder_inputs:0': input_seq,\n",
    "                                                 'encoder_inputs_length:0': [length],\n",
    "                                                 'letter_targets:0': targets,\n",
    "                                                 'is_training:0': False,\n",
    "                                                 'keep_prob:0': 1})[0]\n",
    "            word = ''\n",
    "            for i in pred:\n",
    "                if word == 1:\n",
    "                    break\n",
    "                else:\n",
    "                    word += idx2char(i, True)\n",
    "\n",
    "        return word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CharCycler(Cycler):\n",
    "    \"\"\" Cycle through the words and recognise them \"\"\" \n",
    "    def recogniseWord(self, img):\n",
    "        img = cv2.copyMakeBorder(img,\n",
    "                                 0, 0, 30, 30,\n",
    "                                 cv2.BORDER_CONSTANT,\n",
    "                                 value=[0, 0, 0])\n",
    "        gaps = characters.segment(img, RNN=True)\n",
    "        \n",
    "        chars = []\n",
    "        for i in range(len(gaps)-1):\n",
    "            char = img[:, gaps[i]:gaps[i+1]]\n",
    "            # TODO None type error after treshold\n",
    "            char, dim = letter_normalization(char, is_thresh=True, dim=True)\n",
    "            # TODO Test different values\n",
    "            if dim[0] > 4 and dim[1] > 4:\n",
    "                chars.append(char.flatten())\n",
    "                \n",
    "        chars = np.array(chars)\n",
    "        word = ''\n",
    "        if len(chars) != 0:\n",
    "            if self.charRNN:\n",
    "                pred = self.charClass.eval_feed({'inputs:0': [chars],\n",
    "                                                 'length:0': [len(chars)],\n",
    "                                                 'keep_prob:0': 1})[0]\n",
    "            else:\n",
    "                pred = self.charClass.run(chars)\n",
    "                \n",
    "            for c in pred:\n",
    "                # word += CHARS[charIdx]\n",
    "                word += idx2char(c)        \n",
    "        return word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "STATS: Seq2Seq\n",
      "spreads : zpaside\n",
      "--- 68.82 seconds ---\n",
      "Correct/Total: 641 / 1356\n",
      "CERacc: 54.351 %\n",
      "Letter Accuracy: 47.2714 %\n",
      "Letter Accuracy with Correction: 46.2389 %\n",
      "Word Accuracy: 23.5955 %\n",
      "Word Accuracy with Correction: 31.8352 %\n",
      "\n",
      "STATS: Seq2SeqX\n",
      "spreads : spreads\n",
      "--- 547.08 seconds ---\n",
      "Correct/Total: 979 / 1356\n",
      "CERacc: 76.3274 %\n",
      "Letter Accuracy: 72.1976 %\n",
      "Letter Accuracy with Correction: 76.2537 %\n",
      "Word Accuracy: 40.4494 %\n",
      "Word Accuracy with Correction: 64.0449 %\n",
      "\n",
      "STATS: CTC\n",
      "spreads : spreads\n",
      "--- 761.05 seconds ---\n",
      "Correct/Total: 985 / 1356\n",
      "CERacc: 82.9646 %\n",
      "Letter Accuracy: 72.6401 %\n",
      "Letter Accuracy with Correction: 77.8761 %\n",
      "Word Accuracy: 49.8127 %\n",
      "Word Accuracy with Correction: 68.9139 %\n",
      "\n",
      "STATS: Bi-RNN and CNN\n",
      "spreads : spreads\n",
      "--- 581.9 seconds ---\n",
      "Correct/Total: 1057 / 1356\n",
      "CERacc: 84.4395 %\n",
      "Letter Accuracy: 77.9499 %\n",
      "Letter Accuracy with Correction: 82.2271 %\n",
      "Word Accuracy: 64.0449 %\n",
      "Word Accuracy with Correction: 74.5318 %\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<__main__.CharCycler at 0x7f5d1ddf8d30>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Class cycling through words\n",
    "\n",
    "WordCycler(images,\n",
    "           labels,\n",
    "           wordClass,\n",
    "           stats='Seq2Seq',\n",
    "           slider=(60, 2),\n",
    "           seq2seq=True)\n",
    "\n",
    "WordCycler(images,\n",
    "           labels,\n",
    "           wordClass2,\n",
    "           stats='Seq2SeqX',\n",
    "           slider=(60, 2))\n",
    "\n",
    "WordCycler(images,\n",
    "           labels,\n",
    "           wordClass3,\n",
    "           stats='CTC',\n",
    "           slider=(60, 60),\n",
    "           ctc=True)\n",
    "\n",
    "CharCycler(images,\n",
    "           labels,\n",
    "           charClass_1,\n",
    "           stats='Bi-RNN and CNN',\n",
    "           charRNN=False)\n",
    "\n",
    "# Cycler(images,\n",
    "#        labels,\n",
    "#        charClass_2,\n",
    "#        charRNN=True)\n",
    "\n",
    "# Cycler(images,\n",
    "#        labels,\n",
    "#        charClass_3,\n",
    "#        charRNN=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}