word2vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7fe1e9435768>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# see http://pytorch.org/tutorials/beginner/nlp/word_embeddings_tutorial.html\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "torch.manual_seed(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_size: 49\n"
     ]
    }
   ],
   "source": [
    "CONTEXT_SIZE = 2  # 2 words to the left, 2 to the right\n",
    "text = \"\"\"We are about to study the idea of a computational process.\n",
    "Computational processes are abstract beings that inhabit computers.\n",
    "As they evolve, processes manipulate other abstract things called data.\n",
    "The evolution of a process is directed by a pattern of rules\n",
    "called a program. People create programs to direct processes. In effect,\n",
    "we conjure the spirits of the computer with our spells.\"\"\".split()\n",
    "\n",
    "split_ind = (int)(len(text) * 0.8)\n",
    "\n",
    "# By deriving a set from `raw_text`, we deduplicate the array\n",
    "vocab = set(text)\n",
    "vocab_size = len(vocab)\n",
    "print('vocab_size:', vocab_size)\n",
    "\n",
    "w2i = {w: i for i, w in enumerate(vocab)}\n",
    "i2w = {i: w for i, w in enumerate(vocab)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cbow sample (['We', 'are', 'to', 'study'], 'about')\n",
      "skipgram sample ('about', 'We', 1)\n"
     ]
    }
   ],
   "source": [
    "# context window size is two\n",
    "def create_cbow_dataset(text):\n",
    "    data = []\n",
    "    for i in range(2, len(text) - 2):\n",
    "        context = [text[i - 2], text[i - 1],\n",
    "                   text[i + 1], text[i + 2]]\n",
    "        target = text[i]\n",
    "        data.append((context, target))\n",
    "    return data\n",
    "\n",
    "def create_skipgram_dataset(text):\n",
    "    import random\n",
    "    data = []\n",
    "    for i in range(2, len(text) - 2):\n",
    "        data.append((text[i], text[i-2], 1))\n",
    "        data.append((text[i], text[i-1], 1))\n",
    "        data.append((text[i], text[i+1], 1))\n",
    "        data.append((text[i], text[i+2], 1))\n",
    "        # negative sampling\n",
    "        for _ in range(4):\n",
    "            if random.random() < 0.5 or i >= len(text) - 3:\n",
    "                rand_id = random.randint(0, i-1)\n",
    "            else:\n",
    "                rand_id = random.randint(i+3, len(text)-1)\n",
    "            data.append((text[i], text[rand_id], 0))\n",
    "    return data\n",
    "\n",
    "cbow_train = create_cbow_dataset(text)\n",
    "skipgram_train = create_skipgram_dataset(text)\n",
    "print('cbow sample', cbow_train[0])\n",
    "print('skipgram sample', skipgram_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CBOW(nn.Module):\n",
    "    def __init__(self, vocab_size, embd_size, context_size, hidden_size):\n",
    "        super(CBOW, self).__init__()\n",
    "        self.embeddings = nn.Embedding(vocab_size, embd_size)\n",
    "        self.linear1 = nn.Linear(2*context_size*embd_size, hidden_size)\n",
    "        self.linear2 = nn.Linear(hidden_size, vocab_size)\n",
    "        \n",
    "    def forward(self, inputs):\n",
    "        embedded = self.embeddings(inputs).view((1, -1))\n",
    "        hid = F.relu(self.linear1(embedded))\n",
    "        out = self.linear2(hid)\n",
    "        log_probs = F.log_softmax(out)\n",
    "        return log_probs\n",
    "\n",
    "class SkipGram(nn.Module):\n",
    "    def __init__(self, vocab_size, embd_size):\n",
    "        super(SkipGram, self).__init__()\n",
    "        self.embeddings = nn.Embedding(vocab_size, embd_size)\n",
    "    \n",
    "    def forward(self, focus, context):\n",
    "        embed_focus = self.embeddings(focus).view((1, -1))\n",
    "        embed_ctx = self.embeddings(context).view((1, -1))\n",
    "        score = torch.mm(embed_focus, torch.t(embed_ctx))\n",
    "        log_probs = F.logsigmoid(score)\n",
    "    \n",
    "        return log_probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CBOW (\n",
      "  (embeddings): Embedding(49, 100)\n",
      "  (linear1): Linear (400 -> 64)\n",
      "  (linear2): Linear (64 -> 49)\n",
      ")\n",
      "SkipGram (\n",
      "  (embeddings): Embedding(49, 100)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "embd_size = 100\n",
    "learning_rate = 0.001\n",
    "n_epoch = 30\n",
    "\n",
    "def train_cbow():\n",
    "    hidden_size = 64\n",
    "    losses = []\n",
    "    loss_fn = nn.NLLLoss()\n",
    "    model = CBOW(vocab_size, embd_size, CONTEXT_SIZE, hidden_size)\n",
    "    print(model)\n",
    "    optimizer = optim.SGD(model.parameters(), lr=learning_rate)\n",
    "\n",
    "    for epoch in range(n_epoch):\n",
    "        total_loss = .0\n",
    "        for context, target in cbow_train:\n",
    "            ctx_idxs = [w2i[w] for w in context]\n",
    "            ctx_var = Variable(torch.LongTensor(ctx_idxs))\n",
    "\n",
    "            model.zero_grad()\n",
    "            log_probs = model(ctx_var)\n",
    "\n",
    "            loss = loss_fn(log_probs, Variable(torch.LongTensor([w2i[target]])))\n",
    "\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            total_loss += loss.data[0]\n",
    "        losses.append(total_loss)\n",
    "    return model, losses\n",
    "\n",
    "def train_skipgram():\n",
    "    losses = []\n",
    "    loss_fn = nn.MSELoss()\n",
    "    model = SkipGram(vocab_size, embd_size)\n",
    "    print(model)\n",
    "    optimizer = optim.SGD(model.parameters(), lr=learning_rate)\n",
    "    \n",
    "    for epoch in range(n_epoch):\n",
    "        total_loss = .0\n",
    "        for in_w, out_w, target in skipgram_train:\n",
    "            in_w_var = Variable(torch.LongTensor([w2i[in_w]]))\n",
    "            out_w_var = Variable(torch.LongTensor([w2i[out_w]]))\n",
    "            \n",
    "            model.zero_grad()\n",
    "            log_probs = model(in_w_var, out_w_var)\n",
    "            loss = loss_fn(log_probs[0], Variable(torch.Tensor([target])))\n",
    "            \n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            total_loss += loss.data[0]\n",
    "        losses.append(total_loss)\n",
    "    return model, losses\n",
    "    \n",
    "cbow_model, cbow_losses = train_cbow()\n",
    "sg_model, sg_losses = train_skipgram()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====Test CBOW===\n",
      "predicted: about\n",
      "label    : about\n",
      "predicted: to\n",
      "label    : to\n",
      "predicted: study\n",
      "label    : study\n",
      "predicted: the\n",
      "label    : the\n",
      "predicted: idea\n",
      "label    : idea\n",
      "predicted: of\n",
      "label    : of\n",
      "predicted: a\n",
      "label    : a\n",
      "predicted: computational\n",
      "label    : computational\n",
      "predicted: process.\n",
      "label    : process.\n",
      "predicted: Computational\n",
      "label    : Computational\n",
      "predicted: processes\n",
      "label    : processes\n",
      "predicted: are\n",
      "label    : are\n",
      "predicted: abstract\n",
      "label    : abstract\n",
      "predicted: beings\n",
      "label    : beings\n",
      "predicted: that\n",
      "label    : that\n",
      "predicted: inhabit\n",
      "label    : inhabit\n",
      "predicted: computers.\n",
      "label    : computers.\n",
      "predicted: As\n",
      "label    : As\n",
      "predicted: they\n",
      "label    : they\n",
      "predicted: evolve,\n",
      "label    : evolve,\n",
      "predicted: processes\n",
      "label    : processes\n",
      "predicted: manipulate\n",
      "label    : manipulate\n",
      "predicted: other\n",
      "label    : other\n",
      "predicted: abstract\n",
      "label    : abstract\n",
      "predicted: things\n",
      "label    : things\n",
      "predicted: called\n",
      "label    : called\n",
      "predicted: data.\n",
      "label    : data.\n",
      "predicted: The\n",
      "label    : The\n",
      "predicted: evolution\n",
      "label    : evolution\n",
      "predicted: of\n",
      "label    : of\n",
      "predicted: a\n",
      "label    : a\n",
      "predicted: process\n",
      "label    : process\n",
      "predicted: is\n",
      "label    : is\n",
      "predicted: directed\n",
      "label    : directed\n",
      "predicted: by\n",
      "label    : by\n",
      "predicted: a\n",
      "label    : a\n",
      "predicted: pattern\n",
      "label    : pattern\n",
      "predicted: of\n",
      "label    : of\n",
      "predicted: rules\n",
      "label    : rules\n",
      "predicted: called\n",
      "label    : called\n",
      "predicted: a\n",
      "label    : a\n",
      "predicted: program.\n",
      "label    : program.\n",
      "predicted: People\n",
      "label    : People\n",
      "predicted: create\n",
      "label    : create\n",
      "predicted: programs\n",
      "label    : programs\n",
      "predicted: to\n",
      "label    : to\n",
      "predicted: direct\n",
      "label    : direct\n",
      "predicted: processes.\n",
      "label    : processes.\n",
      "predicted: In\n",
      "label    : In\n",
      "predicted: effect,\n",
      "label    : effect,\n",
      "predicted: a\n",
      "label    : we\n",
      "predicted: conjure\n",
      "label    : conjure\n",
      "predicted: the\n",
      "label    : the\n",
      "predicted: spirits\n",
      "label    : spirits\n",
      "predicted: of\n",
      "label    : of\n",
      "predicted: the\n",
      "label    : the\n",
      "predicted: computer\n",
      "label    : computer\n",
      "predicted: with\n",
      "label    : with\n",
      "Accuracy: 98.3% (57/58)\n",
      "------\n",
      "====Test SkipGram===\n",
      "Accuracy: 50.0% (232/464)\n"
     ]
    }
   ],
   "source": [
    "# test\n",
    "# You have to use other dataset for test, but in this case I use training data because this dataset is too small\n",
    "def test_cbow(test_data, model):\n",
    "    print('====Test CBOW===')\n",
    "    correct_ct = 0\n",
    "    for ctx, target in test_data:\n",
    "        ctx_idxs = [w2i[w] for w in ctx]\n",
    "        ctx_var = Variable(torch.LongTensor(ctx_idxs))\n",
    "\n",
    "        model.zero_grad()\n",
    "        log_probs = model(ctx_var)\n",
    "        _, predicted = torch.max(log_probs.data, 1)\n",
    "        predicted_word = i2w[predicted[0]]\n",
    "        print('predicted:', predicted_word)\n",
    "        print('label    :', target)\n",
    "        if predicted_word == target:\n",
    "            correct_ct += 1\n",
    "            \n",
    "    print('Accuracy: {:.1f}% ({:d}/{:d})'.format(correct_ct/len(test_data)*100, correct_ct, len(test_data)))\n",
    "\n",
    "def test_skipgram(test_data, model):\n",
    "    print('====Test SkipGram===')\n",
    "    correct_ct = 0\n",
    "    for in_w, out_w, target in test_data:\n",
    "        in_w_var = Variable(torch.LongTensor([w2i[in_w]]))\n",
    "        out_w_var = Variable(torch.LongTensor([w2i[out_w]]))\n",
    "\n",
    "        model.zero_grad()\n",
    "        log_probs = model(in_w_var, out_w_var)\n",
    "        _, predicted = torch.max(log_probs.data, 1)\n",
    "        predicted = predicted[0]\n",
    "        if predicted == target:\n",
    "            correct_ct += 1\n",
    "\n",
    "    print('Accuracy: {:.1f}% ({:d}/{:d})'.format(correct_ct/len(test_data)*100, correct_ct, len(test_data)))\n",
    "\n",
    "test_cbow(cbow_train, cbow_model)\n",
    "print('------')\n",
    "test_skipgram(skipgram_train, sg_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe1a5910f28>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd0FdX6xvHvm0KQ0CEghtCLdIRQJCRRkS6iqFwUG6hY\nUAlwf1779aq3qYQigqIioiiiVFGpIgmd0DuEHqSDFFHq/v2RozeimBASTsnzWcvFZJ+ZnHfWyOO4\nZ8/e5pxDREQCV5C3CxARkdyloBcRCXAKehGRAKegFxEJcAp6EZEAp6AXEQlwCnoRkQCnoBcRCXAK\nehGRABfi7QIASpYs6SpUqODtMkRE/MqSJUsOOOciMtvPJ4K+QoUKpKSkeLsMERG/Ymbbs7Kfum5E\nRAKcgl5EJMAp6EVEApyCXkQkwCnoRUQCnIJeRCTAKehFRAKcXwf9z6fP8tKkNRw4ftLbpYiI+Cy/\nDvoVO3/gk0U7aDcwmfmbD3q7HBERn+TXQd+kUgkm9oyhYP4Qur63gP7TN3L2nBY7FxHJyK+DHqBG\nmcJ8+Xhzbr2mLANnbuKudxew58jP3i5LRMRn+H3QA4SHhdCvcz363VGPVbuO0G5QMrM27PN2WSIi\nPiEggv4XtzUsy6THm1OqUBjdPljMv79ex+mz57xdloiIVwVU0ANUKVWQCT1juLtpOd5J2kLnd+az\n89AJb5clIuI1ARf0APlDg3n1ljq8dVcDUvcep/2gZKas3u3tskREvCIgg/4X7euW4asnY6lYMpxH\nPl7KixNX8/Pps94uS0TksgrooAcoV6IAnz/SjIdiKzJy/nY6Dp7Lxr3HvF2WiMhlk2nQm1mUmc0y\ns7VmtsbMennaXzez9Wa20szGm1nRDMc8Y2apZrbBzFrn5glkRb6QIJ5rX5MR3Rpx8MeTdHhzDh8v\n2I5zGnMvIoEvK3f0Z4C+zrmaQFOgp5nVBKYDtZ1zdYGNwDMAns+6ALWANsAQMwvOjeIv1nXVS/FN\nrziaVCrB8xNW8/BHS/jhxClvlyUikqsyDXrn3G7n3FLP9jFgHRDpnJvmnDvj2W0BUNaz3REY7Zw7\n6ZzbCqQCjXO+9OyJKBTGiPsb8Xz7GszasI+2A5NZsEXTJ4hI4LqoPnozqwBcAyw876PuwDee7Uhg\nZ4bP0jxtPiMoyHgwthLjH4shf2gwd767gH7TNnBGY+5FJABlOejNrCAwFkhwzh3N0P4c6d07oy7m\ni82sh5mlmFnK/v37L+bQHFM7sgiTn2jO7Q3K8ua3qRpzLyIBKUtBb2ahpIf8KOfcuAzt9wM3AV3d\n/55s7gKiMhxe1tP2G865Yc65aOdcdERERDbLv3ThYSG8fkc9Bt15DZv2HqfdwGQmrfjea/WIiOS0\nrIy6MeB9YJ1zLjFDexvgKeBm51zG2+BJQBczCzOzikBVYFHOlp3zbq53FV/3iqVK6YI8+eky+oxZ\nzrGfT3u7LBGRS5aVO/oY4B7gBjNb7vmnHTAYKARM97S9DeCcWwOMAdYCU4Cezjm/eEspqngBxjx8\nLU/eUIUJy3bRdmAyi7Ye8nZZIiKXxHxhLHl0dLRLSUnxdhm/sWT7YXp/tpydh0/waHxlEm6sRr6Q\ngH+/TET8iJktcc5FZ7afkusCGpYvxte9YuncMIoh322m09C5pO7TG7Ui4n8U9H+iYFgI/729Lm/f\n3ZBdh3+i/aA5jJy/TW/UiohfUdBnQZvaVzI1IY6mlUrw4sQ13P/BYvYd1SpWIuIfFPRZVKpwfkZ0\na8TLHWuxYMtBWg9IYsrqPd4uS0QkUwr6i2Bm3HttBb56sjmRxa7gkY+X8NQXKzQMU0R8moI+G6qU\nKsS4R2N47LrKfLEkjTYDkpm/WfPliIhvUtBnU76QIJ5qczWfP3ItocHGne8u4OUv12phExHxOQr6\nS9SwfHG+7hXLPU3LM3zuVtoPSmZl2g/eLktE5FcK+hxQIF8Ir9xSm5HdG/PjybPcOmQe/adv5LRm\nwxQRH6Cgz0Fx1SKYmhDHzfWuYuDMTXQaMo9NWrZQRLxMQZ/DihQIpf9f6jO0awPSDp+g/ZtzeC95\nC+fO6SUrEfEOBX0uaVunDNN6xxNXNYJXv1pHl3cXsOOg5roXkctPQZ+LIgqF8e69DXn99rqs/f4o\nbQYm8dGC7bq7F5HLSkGfy8yMO6KjmNo7jobli/HChNXcM3whaYd1dy8il4eC/jKJLHoFI7s35p+3\n1mbZjh9oMyCZ0Yt2aII0Ecl1CvrLyMzo2qQ8UxPiqBNZhKfHreK+Dxaz+8hP3i5NRAKYgt4LoooX\nYNSDTXi5Yy0Wbz1Eq/5JfLEkTXf3IpIrFPReEhSUPkHalIRYalxZmL9+voIHP0xhr6Y/FpEcpqD3\nsvIlwhndoykv3FSTOakHaNU/ifHLdHcvIjlHQe8DgoKMB5pX5JtesVSOCKf3Z+l393uO6O5eRC6d\ngt6HVIooyOePNOOFm2oyd/MBWibO5rPFGpkjIpdGQe9jgj1391N6xVHzqsL8bewq7nl/ETsPady9\niGSPgt5HVSgZzqcPNeWVW2qzbMdhWg9IYuT8bXqrVkQumoLehwUFGfc0Lf/rW7UvTlxDl2EL2Hrg\nR2+XJiJ+REHvB8oWK8DI7o157fa6rNtzlDYDkng3aQtndXcvIlmgoPcTZkbn6Chm9IkntmpJ/vn1\nOm4bOo+Nmu9eRDKRadCbWZSZzTKztWa2xsx6edqLm9l0M9vk+bNYhmOeMbNUM9tgZq1z8wTymtKF\n8/PuvdEM7FKf7Qd/pP2gZAbM2MipM1rNSkT+WFbu6M8AfZ1zNYGmQE8zqwk8Dcx0zlUFZnp+xvNZ\nF6AW0AYYYmbBuVF8XmVmdKwfyYw+8bStXYYBMzbR4c05LN+ptWpF5PcyDXrn3G7n3FLP9jFgHRAJ\ndAQ+9Oz2IXCLZ7sjMNo5d9I5txVIBRrndOECJQqGMejOa3j/vmiO/HSaTkPm8srktZw4dcbbpYmI\nD7moPnozqwBcAywESjvndns+2gOU9mxHAjszHJbmaZNc0qJGaab1iePOxuV4f85WWg9IYm7qAW+X\nJSI+IstBb2YFgbFAgnPuaMbPXPqrmxc1BMTMephZipml7N+//2IOlT9QOH8o/7y1DqN7NCUkKIiu\n7y3kqS9WcOTEaW+XJiJelqWgN7NQ0kN+lHNunKd5r5mV8XxeBtjnad8FRGU4vKyn7Tecc8Occ9HO\nueiIiIjs1i/naVqpBN/0iuWR+MqMXbqLG/vPZsrq3ZkfKCIBKyujbgx4H1jnnEvM8NEk4D7P9n3A\nxAztXcwszMwqAlWBRTlXsmQmf2gwT7e9mok9Y4goGMYjHy/lkY+WaApkkTzKMpswy8yaA8nAKuCX\nMXzPkt5PPwYoB2wHOjvnDnmOeQ7oTvqInQTn3Dd/9h3R0dEuJSXlEk5DLuT02XMMS9rCwJmbCAsO\n4qm2V9O1cTmCgszbpYnIJTKzJc656Ez384WZERX0uW/bgR95bsIq5qYepEG5ovy7U12qX1nI22WJ\nyCXIatDrzdg8okLJcD5+oAn97qjH1gPpL1q9MXUDP58+6+3SRCSXKejzEDPjtoZlmdn3Om6ufxWD\nZ6XSZkAS8zQUUySgKejzoOLh+UjsXJ+PH2iCA+56byF//XwFh3885e3SRCQXKOjzsOZVSzI1IY7H\nrqvMhGW7aJE4W+vVigQgBX0elz80mKfaXM3kJ5tTrngBen+2gnveX8Q2zXkvEjAU9ALA1VcWZuyj\nzXi5Yy1W7PyBVgOSGPztJs2KKRIAFPTyq+Ag495rKzCjbzwta5TmjWkbaT8omcXbDnm7NBG5BAp6\n+Z3ShfPzVtcGvH9fNCdOneWOt+fzzLiV/HBCD2tF/JGCXi6oRY3STO8TR4+4SoxJSePGxNlMXL5L\nD2tF/IyCXv5UgXwhPNuuBpMejyGy6BX0Gr2ce4cvYvtBPawV8RcKesmSWlcVYdxjMfzj5los2/ED\nrfqnP6w9eUZv1or4OgW9ZFlwkHFfswrM6BPPDVeX4o1pG2k7MFlv1or4OAW9XLQri+Rn6N0N+eD+\nRpw567jrvYX0Gr2MfZoGWcQnKegl266/uhTTesfxZIuqfLNqDy36zWbE3K2cPaeHtSK+REEvlyR/\naDB9WlZjau846pcryktfruXmwXNYtuOwt0sTEQ8FveSIiiXDGdm9MW/d1YADx0/Saeg8nh2/SmPv\nRXyAgl5yjJnRvm4ZZva9jgdiKvLZ4p3c0G82n6fs1Nh7ES9S0EuOKxgWwvM31WTyE82pWDKc//ti\nJZ3fmc+GPce8XZpInqSgl1xTo0xhPn/4Wl67rS6p+47TflAy//56HT+ePOPt0kTyFAW95KqgIKNz\noyi+7XsdtzcsyztJW2iZOJspq/eoO0fkMlHQy2VRLDwf/7mtLmMfvZbCV4TyyMdLeODDFHYeOuHt\n0kQCnoJeLquG5Ysz+YnmPN++Bgu3HKRl/9m8NStV896L5CIFvVx2IcFBPBhbiRl906dSeH3qBtoO\n1CLlIrlFQS9eU6bIFQzp2pAPujXitGcqhYTRy9h/7KS3SxMJKAp68brrq3umUrihCl+v2sMN/b7j\nowXbNZWCSA5R0ItPyB8aTJ9W1fkmIZY6kUV4YcJqOg2dx+pdR7xdmojfyzTozWy4me0zs9UZ2uqb\n2QIzW25mKWbWOMNnz5hZqpltMLPWuVW4BKbKEQUZ9WATBnapz67DJ7h58BxemrSGYz+f9nZpIn4r\nK3f0I4A257W9BvzDOVcfeNHzM2ZWE+gC1PIcM8TMgnOsWskTzIyO9SOZ2fc6ujYpz4fzt9Gi32wm\nr/xeY+9FsiHToHfOJQGHzm8GCnu2iwDfe7Y7AqOdcyedc1uBVKAxItlQ5IpQXrmlNuMfiyGiUBiP\nf7KM+z5YrGUMRS5SdvvoE4DXzWwn8AbwjKc9EtiZYb80T9vvmFkPT7dPyv79+7NZhuQF9aOKMrFn\nDH/vUJOl2w/Tsn8Sg2ZqGUORrMpu0D8K9HbORQG9gfcv9hc454Y556Kdc9ERERHZLEPyipDgILrF\nVGRm33ha1SxN4vSNtB2gZQxFsiK7QX8fMM6z/Tn/657ZBURl2K+sp00kR5QunJ/BdzXgw+6NOXMu\nfex978+Wa+y9yJ/IbtB/D8R7tm8ANnm2JwFdzCzMzCoCVYFFl1aiyO/FV4v4dez95JXf06Lfd4xa\nuJ1zGnsv8jtZGV75KTAfqG5maWb2APAQ0M/MVgD/AnoAOOfWAGOAtcAUoKdzTh2pkit+HXvfK46a\nVxXmufGrue3teaz9/qi3SxPxKeYLw9Wio6NdSkqKt8sQP+acY/yyXfzzq3X88NNpujWrQO+W1QgP\nC/F2aSK5xsyWOOeiM9tPb8ZKQDAzOjUoy8y+8XSOjuK9OVu5UfPeiwAKegkwRQvk49+d6jD20Wsp\n4pn3/sEPU0g7rHnvJe9S0EtAali+OF8+0Zxn213NvM0HaZmYxDuzN3P6rOa9l7xHQS8BKzQ4iB5x\nlZnRN56YKiX59zfr6fDmHJZsP/9Fb5HApqCXgBdZ9Areuy+aYfc05OhPp7lt6HyeGbeSH06c8nZp\nIpeFgl7yjFa1rmR6n3geiq3ImJQ0WvSbzbilaXpYKwFPQS95SnhYCM+1r8mXjzcnqngB+oxZwV3v\nLmTz/uPeLk0k1yjoJU+qeVVhxj3ajFdvqc3q74/QdkAyidM38vNpvd8ngUdBL3lWUJBxd9PyfNv3\nOtrWuZJBMzfRdmAyczVRmgQYBb3keRGFwhjY5Ro+fqAJzjm6eiZKO3BcE6VJYFDQi3g0r1qSKQlx\nPPHrRGmz+WzxDk2UJn5PQS+SQf7QYPq2qs43vWKpXroQfxu7ir8Mm8+mvce8XZpItinoRf5AlVKF\nGN2jKa/dVpdN+47TblAyr09dr4e14pcU9CIXEBRkdG4Uxcw+8dxcL5K3Zm2mVf8kkjZq6UvxLwp6\nkUyUKBhGv871+OShJoQEGfcOX8STny7TqlbiNxT0IlnUrHJJvu4VS68WVZmyeg8t+n3Hp4v0sFZ8\nn4Je5CLkDw2md8tqfN0rlhplCvPMOD2sFd+noBfJhiqlCjK6R1Nev/1/D2vfmLpBD2vFJynoRbLJ\nzLgjOv1hbYd6VzF4ViptBiQxZ5PerBXfoqAXuUQlCoaR2Lk+nzzYBDPj7vfT36w9qDdrxUco6EVy\nSLMqJfmmVyxP/vJmbeJsxizeqWmQxesU9CI5KH9oMH08b9ZWK1WIp8aupMuwBZoGWbxKQS+SC355\ns/Y/neqwbvdR2g5IZtDMTZw6ozVr5fJT0IvkkqAgo0vjcszoG0/r2leSOH0j7Qclk7JNa9bK5aWg\nF8llpQrl5807r+GD+xtx4tRZbn97Ps+OX8WRn057uzTJIzINejMbbmb7zGz1ee1PmNl6M1tjZq9l\naH/GzFLNbIOZtc6NokX80fVXl2Ja7zgebF6R0Yt2cGPibL5etVsPayXXZeWOfgTQJmODmV0PdATq\nOedqAW942msCXYBanmOGmFlwThYs4s/Cw0J4/qaaTOzZnFKFwnhs1FIeGpnC9z/85O3SJIBlGvTO\nuSTg/E7FR4H/OOdOevbZ52nvCIx2zp10zm0FUoHGOVivSECoU7YIE3vG8Fy7GsxNPUjLxNl8MHcr\nZzVvjuSC7PbRVwNizWyhmc02s0ae9khgZ4b90jxtInKekOAgHoqrxLTecURXKM4/vlxLp6HzWLf7\nqLdLkwCT3aAPAYoDTYH/A8aYmV3MLzCzHmaWYmYp+/drfm/Ju6KKF2BEt0YM7FKftEMn6PDmHF6b\nokVOJOdkN+jTgHEu3SLgHFAS2AVEZdivrKftd5xzw5xz0c656IiIiGyWIRIYzIyO9SOZ0SeeW66J\nZMh3m2k9IIm5qZo3Ry5ddoN+AnA9gJlVA/IBB4BJQBczCzOzikBVYFFOFCqSFxQLz8cbd9Tjkweb\nAND1vYX89fMVHP7xlJcrE3+WleGVnwLzgepmlmZmDwDDgUqeIZejgfs8d/drgDHAWmAK0NM5p///\nFLlIzaqUZGpCHI9dV5kJy3ZxY+JsJi7fpaGYki3mC//iREdHu5SUFG+XIeKT1u0+ytNjV7Ii7Qjx\n1SJ49ZbaRBUv4O2yxAeY2RLnXHRm++nNWBEfV6NMYcY9FsPfO9Rk8bZDtOqfxHvJWzQUU7JMQS/i\nB4KDjG4xFZneJ56mlYrz6lfr6DR0Huv3aCimZE5BL+JHIotewfD704di7jx0gpsGzaHfNC1hKH9O\nQS/iZzIOxby5/lW8+W0q7Qcls1izYsoFKOhF/FTx8Hwkdq7Ph90b8/Ppc9zx9nxemLCaYz9rVkz5\nLQW9iJ+LrxbBtN5xdI+pyMcLt9OqfxIz1+31dlniQxT0IgEgPCyEFzvUZNyjzSicP5QHPkzh8U+W\nsv+YFigXBb1IQLmmXDG+fKI5fVtWY9qavbTsP5txS9P0olUep6AXCTD5QoJ4okVVvu7VnMoRBekz\nZgXdRixml+a8z7MU9CIBqkqpQnz+8LW81KEmi7YeolXibD6av41zetEqz1HQiwSwoCDj/piKTE2I\no0H5YrwwcQ1dhi1gy/7j3i5NLiMFvUgeEFW8ACO7N+b12+uyfs9R2gxMZuh3mzlz9py3S5PLQEEv\nkkeYGXdERzGjTzzXV4/gv1PWc8uQuaz9XtMoBDoFvUgeU6pwft65J5qhXRuw58hJbh48hzemahqF\nQKagF8mj2tYpw4w+cXSsH8ngWenTKCzZrmkUApGCXiQPK1ogH/0612NEt0b8fPoct789n5cmreHH\nk2e8XZrkIAW9iHBd9VJM7R3HPU3LM2LeNloPSCJ5035vlyU5REEvIgAUDAvh5Y61GfPwtYQGB3HP\n+4t46osVHPlJk6T5OwW9iPxG44rF+aZXLI9eV5mxS3fRMnE2U9fs8XZZcgkU9CLyO/lDg/lbm6uZ\n8FgMJQqG8fBHS+g5SpOk+SsFvYhcUJ2yRZj0eAx/bVWN6WvTJ0kbv0yTpPkbBb2I/KnQ4CAevyF9\nkrRKJcPp/dkKuo9YzO4jmiTNXyjoRSRLqpQqxOePNOOFm2oyf8tBWiUm8cnCHbq79wMKehHJsuAg\n44Hm6ZOk1Y4swrPjV9H1vYXsOHjC26XJn1DQi8hFK18inFEPNuFft9ZhZdoRWg9IYvicrZzVFMg+\nSUEvItkSFGTc1aQc03rH0aRScV6evJbO78wndZ+mQPY1mQa9mQ03s31mtvoPPutrZs7MSmZoe8bM\nUs1sg5m1zumCRcS3XFX0Cj64vxGJneuRuu847QYlM+S7VE2B7EOyckc/AmhzfqOZRQGtgB0Z2moC\nXYBanmOGmFlwjlQqIj7LzOjUoCzT+8TR4upSvDZlA7cOmce63ZoC2RdkGvTOuSTgj6a06w88BWTs\nlOsIjHbOnXTObQVSgcY5UaiI+L5ShfIz9O6GDOnagN1HfqLDm3PoP30jp87o7t6bstVHb2YdgV3O\nuRXnfRQJ7Mzwc5qn7Y9+Rw8zSzGzlP37NXmSSCBpV6cM03rHc1PdMgycuYmbB89hVdoRb5eVZ110\n0JtZAeBZ4MVL+WLn3DDnXLRzLjoiIuJSfpWI+KDi4fkY0OUa3rs3msMnTnHLkLn8d8p6LXDiBdm5\no68MVARWmNk2oCyw1MyuBHYBURn2LetpE5E86saapZnWO57bGkQy9LvNngVODnu7rDzlooPeObfK\nOVfKOVfBOVeB9O6ZBs65PcAkoIuZhZlZRaAqsChHKxYRv1PkilBeu70eI7s39ixwMo9XJq/lp1O6\nu78csjK88lNgPlDdzNLM7IEL7eucWwOMAdYCU4CezjldSREBIK5aBFMSYunapBzvz9lKm4FJLNhy\n0NtlBTzzhXkqoqOjXUpKirfLEJHLaP7mg/xt7Ep2HDrBPU3L87e2V1MwLMTbZfkVM1vinIvObD+9\nGSsiXnFt5RJMSYilW0wFPl64ndb9tXxhblHQi4jXFMgXwt871OKLR64lLDR9+cKnx67k6M9avjAn\nKehFxOsali/O10/G8kh8Zcak7KRVYhLfrt/r7bIChoJeRHxC/tBgnm57NeMfi6HIFaF0H5FC78+W\n88OJU94uze8p6EXEp9SLKsqkJ2J4skVVvlzxPTcmJjFl9W5vl+XXFPQi4nPCQoLp07IaEx+PoXTh\nMB75eCk9Ry3lwHEtTp4dCnoR8Vm1rirChJ4x/F/r6umLkyfOZuLyXVq+8CIp6EXEp4UGB9Hz+ip8\n9WRzypcIp9fo5Tw0cgl7j/7s7dL8hoJeRPxC1dKFGPtoM55vX4PkTfu5MXE2Y1J26u4+CxT0IuI3\ngoOMB2MrMSUhjhplCvPUFyu5d/gi0g5rcfI/o6AXEb9TsWQ4ox9qyisda7Fk+2Fa90/io/nbOKfF\nyf+Qgl5E/FJQkHHPtRWYmhBHg/LFeGHiGu58dwHbDvzo7dJ8joJeRPxaVPECjOzemNduq8va3Udp\nMzCJ95K3cFZ3979S0IuI3zMzOjeKYkafeJpXKcmrX63j9rfnsWnvMW+X5hMU9CISMEoXzs+790Yz\nsEt9th34kfaD5vDWrFROn83bi5Mr6EUkoJgZHetHMr1PPC1rleb1qRvoOHgua77Pu4uTK+hFJCCV\nLBjGW3c14O27G7Lv2Ek6Dp5Lv2kbOHkm7y16p6AXkYDWpvaVzOgTR8f6kbz5bSo3DZrDsh15a3Fy\nBb2IBLyiBfLRr3M9PujWiOMnz3Db0Hn86+t1eWZxcgW9iOQZ11cvxbTecXRpXI5hSVtoOzCJhXlg\ncXIFvYjkKYXyh/KvW+vwyUNNOOfgL8MW8PyEVRwL4OULFfQikic1q1ySKQmxdI+pyKiFO2jVP4lZ\n6/d5u6xcoaAXkTyrQL4QXuxQk7GPNqNQ/hC6jVhMwuhlHPoxsJYvVNCLSJ7XoFwxJj8RS8KNVflq\n1W5uTJzNpBXfB8wUyAp6EREgX0gQCTdWY/ITsUQVL8CTny7jwQ9T2H3kJ2+XdskyDXozG25m+8xs\ndYa2181svZmtNLPxZlY0w2fPmFmqmW0ws9a5VbiISG6ofmUhxnkWOJm7+QAtE5MYtXC7X0+BnJU7\n+hFAm/PapgO1nXN1gY3AMwBmVhPoAtTyHDPEzIJzrFoRkcvglwVOpiXEU7dsEZ4bv5o7313AVj+d\nAjnToHfOJQGHzmub5pw74/lxAVDWs90RGO2cO+mc2wqkAo1zsF4RkcumXIkCjHqwCf+9rU76FMgD\nknh79mbO+NkkaTnRR98d+MazHQnszPBZmqdNRMQvmRl/aVSOGX3iia8WwX++Wc8tQ/xrkrRLCnoz\new44A4zKxrE9zCzFzFL2799/KWWIiOS60oXz8849DRnStQF7jpzk5sFzeX3qen4+7fvTKGQ76M3s\nfuAmoKv73xikXUBUht3Ketp+xzk3zDkX7ZyLjoiIyG4ZIiKXjZnRrk4ZZvSJ49ZrInlr1mbaDUpm\n8bZDmR/sRdkKejNrAzwF3Oycy7j8+iSgi5mFmVlFoCqw6NLLFBHxHUUL5OONO+oxsntjTp05xx1v\nz+fFias5fvJM5gd7QVaGV34KzAeqm1mamT0ADAYKAdPNbLmZvQ3gnFsDjAHWAlOAns453///GhGR\nbIirFsHUhDi6xVTgowXbaZU42yenUTBfePMrOjrapaSkeLsMEZFsW7L9ME+PXcmmfce5pf5VvNih\nFsXD8+Xqd5rZEudcdGb76c1YEZEc0LB8MSY/2ZxeLf43jcKEZbt8YhoFBb2ISA4JCwmmd8v0aRTK\nFS9AwmfLue+Dxew8dCLzg3ORgl5EJIdVv7IQYx9txksdapKy7RCt+ifx/pytnPXSNAoKehGRXBAc\nZNwfU5HpfeJpWqk4r0xeS6chc1m3++hlr0VBLyKSiyKLXsHw+xsxsEt90g7/RIc35/DG1A2X9UUr\nBb2ISC4zMzrWj2RGn3g61o9k8KxU2g1MZsFlWq9WQS8icpkUC89Hv871+OiBxpw6e44uwxbw6uS1\nuf69CnoRkcsstmoE03rH8VBsRcqXDM/17wvJ9W8QEZHfKZAvhOfa17ws36U7ehGRAKegFxEJcAp6\nEZEAp6COISH6AAADxElEQVQXEQlwCnoRkQCnoBcRCXAKehGRAKegFxEJcD6xwpSZ7Qe2X8KvKAkc\nyKFyfIHOx/cF2jkF2vlA4J3TH51PeedcRGYH+kTQXyozS8nKclr+Qufj+wLtnALtfCDwzulSzkdd\nNyIiAU5BLyIS4AIl6Id5u4AcpvPxfYF2ToF2PhB455Tt8wmIPnoREbmwQLmjFxGRC/DroDezNma2\nwcxSzexpb9eTE8xsm5mtMrPlZpbi7XoulpkNN7N9ZrY6Q1txM5tuZps8fxbzZo0X6wLn9JKZ7fJc\np+Vm1s6bNV4MM4sys1lmttbM1phZL0+7X16nPzkff75G+c1skZmt8JzTPzzt2bpGftt1Y2bBwEag\nJZAGLAbudM7l/rpcucjMtgHRzjm/HP9rZnHAcWCkc662p+014JBz7j+e/yAXc879zZt1XowLnNNL\nwHHn3BverC07zKwMUMY5t9TMCgFLgFuA+/HD6/Qn59MZ/71GBoQ7546bWSgwB+gFdCIb18if7+gb\nA6nOuS3OuVPAaKCjl2vK85xzScCh85o7Ah96tj8k/S+h37jAOfkt59xu59xSz/YxYB0QiZ9epz85\nH7/l0h33/Bjq+ceRzWvkz0EfCezM8HMafn5xPRwww8yWmFkPbxeTQ0o753Z7tvcApb1ZTA56wsxW\nerp2/KKb43xmVgG4BlhIAFyn884H/PgamVmwmS0H9gHTnXPZvkb+HPSBqrlzrj7QFujp6TYIGC69\nr9A/+wt/ayhQCagP7Ab6ebeci2dmBYGxQIJz7mjGz/zxOv3B+fj1NXLOnfVkQVmgsZnVPu/zLF8j\nfw76XUBUhp/Letr8mnNul+fPfcB40ruo/N1eTz/qL/2p+7xczyVzzu31/EU8B7yLn10nT7/vWGCU\nc26cp9lvr9MfnY+/X6NfOOd+AGYBbcjmNfLnoF8MVDWzimaWD+gCTPJyTZfEzMI9D5Mws3CgFbD6\nz4/yC5OA+zzb9wETvVhLjvjlL5vHrfjRdfI86HsfWOecS8zwkV9epwudj59fowgzK+rZvoL0QSfr\nyeY18ttRNwCe4VIDgGBguHPun14u6ZKYWSXS7+IBQoBP/O2czOxT4DrSZ9rbC/wdmACMAcqRPktp\nZ+ec3zzcvMA5XUd6l4ADtgEPZ+g79Wlm1hxIBlYB5zzNz5Ler+131+lPzudO/Pca1SX9YWsw6Tfk\nY5xzL5tZCbJxjfw66EVEJHP+3HUjIiJZoKAXEQlwCnoRkQCnoBcRCXAKehGRAKegFxEJcAp6EZEA\np6AXEQlw/w8wB1m4OC+RswAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe1a566b0b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe1ac39f2e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHe9JREFUeJzt3XuQXOWd3vHvMzM9Uo8E04M0FkIXCxmZXdBi2UwUObEd\nXNhGuLILdmwssrXgMotwQVx2ZbcSe/OHna1QZW/iJaEqsBaGAC4vlwVjVLWAgy8xcYWLByKQxMWI\nW6Sx0P2GbqOZ+eWPfnvU09NzUU9LrZnzfKq6+vR7znv6PdMlHs77nvMeRQRmZmblmhrdADMzO/04\nHMzMbBiHg5mZDeNwMDOzYRwOZmY2jMPBzMyGcTiYmdkwDgczMxvG4WBmZsO0NLoBtZo9e3YsWrSo\n0c0wM5tUnn/++Z0R0TnWdpM2HBYtWkR3d3ejm2FmNqlIemc827lbyczMhnE4mJnZMA4HMzMbxuFg\nZmbDOBzMzGwYh4OZmQ3jcDAzs2EyFw6/fXs3f/PEqwwM+PGoZmYjyVw4vLh5L7f9rzc4cLSv0U0x\nMzttZS4cCm2tAOw7dKzBLTEzO31lLxzyOQD2Hu5tcEvMzE5f2QuHtmI47PGZg5nZiDIYDsVupb2H\nfOZgZjaSDIZD8cxh32GfOZiZjSRz4dBeGnNwt5KZ2YgyFw655iZmTmthj7uVzMxGlLlwgGLXki9l\nNTMbWWbDYa/HHMzMRpTNcMi3+molM7NRZDIc2ttyHpA2MxvFmOEg6S5J2yVtKCt7QNK69Hpb0rpU\nvkjS4bJ1f1dW52JJ6yVtknSrJKXyaWl/myQ9K2lR/Q9zqA53K5mZjWo8Zw53AyvLCyLiSxGxLCKW\nAQ8DPylb/UZpXUR8taz8duB6YEl6lfZ5HbAnIs4DbgG+V9ORnIBSt5JnZjUzq27McIiIp4Dd1dal\n//u/CrhvtH1ImgucGRHPREQA9wJXptVXAPek5YeAS0tnFSdLoS3HQMB7vZ6Z1cysmomOOXwc2BYR\nr5eVnZu6lH4t6eOpbB6wpWybLamstG4zQET0AfuAWRNs16gGb4Q76K4lM7NqWiZY/2qGnjVsBRZG\nxC5JFwM/lXThBL9jkKTVwGqAhQsX1ryfjtL8Sod7WUhbXdpmZjaV1HzmIKkF+DzwQKksIo5GxK60\n/DzwBvBBoAeYX1Z9fiojvS8o22c7sKvad0bEmojoioiuzs7OWps+OL+Sr1gyM6tuIt1KnwJejYjB\n7iJJnZKa0/JiigPPb0bEVmC/pBVpPOEa4NFUbS1wbVr+AvDLNC5x0gyGg69YMjOrajyXst4HPA2c\nL2mLpOvSqlUMH4j+BPBSurT1IeCrEVEazL4R+CGwieIZxeOp/E5glqRNwL8FvjmB4xmX9ryn7TYz\nG82YYw4RcfUI5V+uUvYwxUtbq23fDSytUn4E+OJY7agndyuZmY0uk3dIl2ZmdTiYmVWXyXCA4uWs\nfo60mVl1mQ2HgudXMjMbUWbDoaPNM7OamY0ks+HQ7sn3zMxGlNlwKOT9NDgzs5FkNxzSmcNJvt/O\nzGxSymw4dLS10j8QHDjqmVnNzCplNhxKM7O6a8nMbLjMhkMhzcy6x1csmZkNk+Fw8BQaZmYjyWw4\ndHhmVjOzEWU2HEozs+5zt5KZ2TAZDofimcMedyuZmQ2T2XBobWliRmuzxxzMzKrIbDhA8Yolz8xq\nZjZcxsPBU2iYmVWT+XDwfQ5mZsON5xnSd0naLmlDWdl3JPVIWpdeny1b9y1JmyS9JumysvKLJa1P\n626VpFQ+TdIDqfxZSYvqe4gjK+RbfSmrmVkV4zlzuBtYWaX8lohYll6PAUi6AFgFXJjq3CapOW1/\nO3A9sCS9Svu8DtgTEecBtwDfq/FYTpi7lczMqhszHCLiKWD3OPd3BXB/RByNiLeATcBySXOBMyPi\nmShOg3ovcGVZnXvS8kPApaWzipPNM7OamVU3kTGHr0l6KXU7daSyecDmsm22pLJ5abmyfEidiOgD\n9gGzqn2hpNWSuiV179ixYwJNLyrkPTOrmVk1tYbD7cBiYBmwFfh+3Vo0iohYExFdEdHV2dk54f21\nt3lmVjOzamoKh4jYFhH9ETEA3AEsT6t6gAVlm85PZT1pubJ8SB1JLUA7sKuWdp2ojjQzq2+EMzMb\nqqZwSGMIJZ8DSlcyrQVWpSuQzqU48PxcRGwF9ktakcYTrgEeLatzbVr+AvDLOEWDAIMzs/pGODOz\nIVrG2kDSfcAlwGxJW4BvA5dIWgYE8DZwA0BEbJT0IPAy0AfcFBH9aVc3UrzyKQ88nl4AdwI/krSJ\n4sD3qnoc2HgUPL+SmVlVY4ZDRFxdpfjOUba/Gbi5Snk3sLRK+RHgi2O142Q4PubgMwczs3LZvkM6\n7zEHM7NqMh0OgzOz+i5pM7MhMh0OUJyZ1fMrmZkNlflwaM97Cg0zs0qZD4eOGTl3K5mZVch8OBTy\nrex1t5KZ2RCZD4f2tpyvVjIzq5D5cCjkPTOrmVmlzIdDR1txZtb3PDOrmdmgzIdD6S5pdy2ZmR2X\n+XAoza/kcDAzO87hUJq22zOzmpkNynw4dLhbycxsmMyHw+CYg2+EMzMb5HAojTkcdLeSmVlJ5sNh\nWkszbZ6Z1cxsiMyHAxTvdfCYg5nZcWOGg6S7JG2XtKGs7D9LelXSS5IekVRI5YskHZa0Lr3+rqzO\nxZLWS9ok6db0LGnS86YfSOXPSlpU/8McXXs+xz5frWRmNmg8Zw53Aysryp4ElkbERcDvgG+VrXsj\nIpal11fLym8HrgeWpFdpn9cBeyLiPOAW4HsnfBQTVGjL+TnSZmZlxgyHiHgK2F1R9j8jojTfxDPA\n/NH2IWkucGZEPBPFSYzuBa5Mq68A7knLDwGXls4qTpVCW84zs5qZlanHmMNXgMfLPp+bupR+Lenj\nqWwesKVsmy2prLRuM0AKnH3ArDq0a9wKba3s84C0mdmglolUlvQfgD7gx6loK7AwInZJuhj4qaQL\nJ9jG8u9bDawGWLhwYb12W5yZ9VBxZtZTfNJiZnZaqvnMQdKXgX8J/GnqKiIijkbErrT8PPAG8EGg\nh6FdT/NTGel9QdpnC9AO7Kr2nRGxJiK6IqKrs7Oz1qYPU2jL0eeZWc3MBtUUDpJWAv8O+JOIOFRW\n3impOS0vpjjw/GZEbAX2S1qRxhOuAR5N1dYC16blLwC/jFP8cIVCPs2v5EFpMzNgHN1Kku4DLgFm\nS9oCfJvi1UnTgCdTN8wz6cqkTwB/LekYMAB8NSJKg9k3UrzyKU9xjKI0TnEn8CNJmygOfK+qy5Gd\ngEKaQmPf4WPFUxgzs4wbMxwi4uoqxXeOsO3DwMMjrOsGllYpPwJ8cax2nEyDM7P6zMHMDPAd0sDx\nM4c9vpzVzAxwOABlD/zx5axmZoDDATg+bfc+nzmYmQEOB6BsZlaPOZiZAQ6HQYW851cyMytxOCTt\nba2emdXMLHE4JB1tOXcrmZklDoek0Jbz1UpmZonDIWnPt3rabjOzxOGQFNqOz8xqZpZ1DoekkC/O\nzHqwt7/RTTEzaziHQ9IxOL+Su5bMzBwOSekuaV+xZGbmcBg0OL+Sw8HMzOFQMjhtt2+EMzNzOJR0\nuFvJzGyQwyE5c7BbyWcOZmZjhoOkuyRtl7ShrOwsSU9Kej29d5St+5akTZJek3RZWfnFktandbem\nZ0kjaZqkB1L5s5IW1fcQx2d6rpl8zjOzmpnB+M4c7gZWVpR9E/hFRCwBfpE+I+kCis+AvjDVuU1S\nc6pzO3A9sCS9Svu8DtgTEecBtwDfq/VgJspTaJiZFY0ZDhHxFLC7ovgK4J60fA9wZVn5/RFxNCLe\nAjYByyXNBc6MiGeieAvyvRV1Svt6CLi0dFZxqhXaWn3mYGZG7WMOcyJia1p+F5iTlucBm8u225LK\n5qXlyvIhdSKiD9gHzKqxXRNSyOc85mBmRh0GpNOZwCmZkEjSakndkrp37NhR9/27W8nMrKjWcNiW\nuopI79tTeQ+woGy7+amsJy1Xlg+pI6kFaAd2VfvSiFgTEV0R0dXZ2Vlj00dW8DMdzMyA2sNhLXBt\nWr4WeLSsfFW6AulcigPPz6UuqP2SVqTxhGsq6pT29QXgl9GgqVEL6WlwnpnVzLKuZawNJN0HXALM\nlrQF+DbwXeBBSdcB7wBXAUTERkkPAi8DfcBNEVGa5vRGilc+5YHH0wvgTuBHkjZRHPheVZcjq0Eh\nn+NYf3Fm1pnTxvzTmJlNWWP+FzAirh5h1aUjbH8zcHOV8m5gaZXyI8AXx2rHqVBoO34jnMPBzLLM\nd0iXac+Xpu32uIOZZZvDoUxpfqV9vmLJzDLO4VCmNDPrHt/rYGYZ53AoU/DMrGZmgMNhiPa8u5XM\nzMDhMMTxmVndrWRm2eZwqFBoy7HH3UpmlnEOhwrteU+hYWbmcKhQaMuxz8+RNrOMczhU6PAzHczM\nHA6VPOZgZuZwGKY975lZzcwcDhUKbcWZWQ/19o+9sZnZFOVwqFCaX8lPhDOzLHM4VCjNzLrnoK9Y\nMrPscjhUKHhmVjMzh0MlT75nZuZwGKYjTdu91zfCmVmG1RwOks6XtK7stV/SNyR9R1JPWflny+p8\nS9ImSa9Juqys/GJJ69O6WyVpogdWq9LMrD5zMLMsqzkcIuK1iFgWEcuAi4FDwCNp9S2ldRHxGICk\nC4BVwIXASuA2Sc1p+9uB64El6bWy1nZN1PRcM9NzTZ6Z1cwyrV7dSpcCb0TEO6NscwVwf0QcjYi3\ngE3AcklzgTMj4pko3nl2L3BlndpVk0LeU2iYWbbVKxxWAfeVff6apJck3SWpI5XNAzaXbbMllc1L\ny5Xlw0haLalbUveOHTvq1PThCm053+dgZpk24XCQ1Ar8CfAPqeh2YDGwDNgKfH+i31ESEWsioisi\nujo7O+u122EKbTl3K5lZptXjzOFy4IWI2AYQEdsioj8iBoA7gOVpux5gQVm9+amsJy1XljeMu5XM\nLOvqEQ5XU9allMYQSj4HbEjLa4FVkqZJOpfiwPNzEbEV2C9pRbpK6Rrg0Tq0q2buVjKzrGuZSGVJ\nM4BPAzeUFf+NpGVAAG+X1kXERkkPAi8DfcBNEVGa3e5G4G4gDzyeXg1TaGtl36FjRAQNvKrWzKxh\nJhQOEXEQmFVR9mejbH8zcHOV8m5g6UTaUk+Fthy9/QMc6u1nxrQJ/YnMzCYl3yFdRSHvmVnNLNsc\nDlUcn1/JVyyZWTY5HKoopPmV9vmKJTPLKIdDFaUzBz9L2syyyuFQRSHvmVnNLNscDlX4mQ5mlnUO\nhypKM7P6aXBmllUOhxEU8q1+jrSZZZbDYQSeQsPMsszhMIL2fM6XsppZZjkcRtDR1uqrlcwssxwO\nIyi05Xyfg5lllsNhBO1tucGZWc3MssbhMIJCvpXe/gEOH+sfe2MzsynG4TCCDt8IZ2YZ5nAYwfH5\nlTwobWbZ43AYQXveM7OaWXZNKBwkvS1pvaR1krpT2VmSnpT0enrvKNv+W5I2SXpN0mVl5Ren/WyS\ndKtOg2dzDs6v5BvhzCyD6nHm8MmIWBYRXenzN4FfRMQS4BfpM5IuAFYBFwIrgdskNac6twPXA0vS\na2Ud2jUhHemZDh5zMLMsOhndSlcA96Tle4Ary8rvj4ijEfEWsAlYLmkucGZEPBPF60bvLavTMB5z\nMLMsm2g4BPBzSc9LWp3K5kTE1rT8LjAnLc8DNpfV3ZLK5qXlyvKGmp5rZua0FjbvPtToppiZnXIt\nE6z/sYjokfQ+4ElJr5avjIiQVLe7yFIArQZYuHBhvXY7ok98cDY/f2U7/QNBc1PDh0HMzE6ZCZ05\nRERPet8OPAIsB7alriLS+/a0eQ+woKz6/FTWk5Yry6t935qI6IqIrs7Ozok0fVxWLp3LzveO8sL/\n23PSv8vM7HRSczhImiHpjNIy8BlgA7AWuDZtdi3waFpeC6ySNE3SuRQHnp9LXVD7Ja1IVyldU1an\noT55fietzU08vv7dRjfFzOyUmsiZwxzgN5JeBJ4D/jEingC+C3xa0uvAp9JnImIj8CDwMvAEcFNE\nlOamuBH4IcVB6jeAxyfQrro5Y3qOjy+Zzc82vus5lswsU2oec4iIN4EPVSnfBVw6Qp2bgZurlHcD\nS2tty8m0cunZ/OLV7azv2cdF8wuNbo6Z2SnhO6TH8Kk/nENzk3h8g7uWzCw7HA5j6JjRykcXz+KJ\nDe5aMrPscDiMw8qlZ/PWzoP8btt7jW6Kmdkp4XAYh89cOAcJHt+wdeyNzcymAIfDOLzvjOl0vb+D\nJzzuYGYZ4XAYp5VL5/Lquwd4a+fBRjfFzOykcziM08qlZwP47MHMMsHhME7zCnkumt/OEx53MLMM\ncDicgJVLz+bFLfvo2Xu40U0xMzupHA4nYOWFxa6ln7lrycymOIfDCVjcOZPz55zhcQczm/IcDido\n5dKz+e07u9l+4Eijm2JmdtI4HE7Q5X90NhHw5MvbGt0UM7OTxuFwgs6fcwaLZrW5a8nMpjSHwwmS\nxMqlc3n6jV3sPdTb6OaYmZ0UDocaXL70bPoGgp+/sn3sjc3MJiGHQw0umt/OOe3TfUOcmU1ZDoca\nSOKypWfz1Os7ee9oX6ObY2ZWdzWHg6QFkn4l6WVJGyV9PZV/R1KPpHXp9dmyOt+StEnSa5IuKyu/\nWNL6tO5WSZrYYZ18ly+dS2/fAL961V1LZjb1TOTMoQ/4i4i4AFgB3CTpgrTulohYll6PAaR1q4AL\ngZXAbZKa0/a3A9cDS9Jr5QTadUpc/P4OZs+c5quWzGxKqjkcImJrRLyQlg8ArwDzRqlyBXB/RByN\niLeATcBySXOBMyPimSg+h/Ne4Mpa23WqNDeJz1w4h1+9tp0jx/ob3Rwzs7qqy5iDpEXAh4FnU9HX\nJL0k6S5JHalsHrC5rNqWVDYvLVeWV/ue1ZK6JXXv2LGjHk2fkMuXns2h3n6e+l3j22JmVk8TDgdJ\nM4GHgW9ExH6KXUSLgWXAVuD7E/2OkohYExFdEdHV2dlZr93WbMXiWbTnc+5aMrMpZ0LhIClHMRh+\nHBE/AYiIbRHRHxEDwB3A8rR5D7CgrPr8VNaTlivLT3u55iY+9YdzePKVbfT2DTS6OWZmdTORq5UE\n3Am8EhF/W1Y+t2yzzwEb0vJaYJWkaZLOpTjw/FxEbAX2S1qR9nkN8Git7TrVLl96NgeO9PH0m7sa\n3RQzs7ppmUDdfw78GbBe0rpU9lfA1ZKWAQG8DdwAEBEbJT0IvEzxSqebIqI0knsjcDeQBx5Pr0nh\nY0tmM6O1mSc2bOVffLDxXV1mZvVQczhExG+AavcjPDZKnZuBm6uUdwNLa21LI03PNfPJP3gfP9u4\njb/8zFFmzZzW6CaZmU2Y75Cug6987FwOHu3j6jue8XMezGxKcDjUwUcWdvA/vvxP2Lz7MKvWPMO2\n/Q4IM5vcHA518s/Om809X1nOtn1H+NIPnub3ew83uklmZjVzONTR8nPP4kd//k/ZdbCXq37wNJt3\nH2p0k8zMauJwqLOPLOzg7/98BQeO9PGlHzzN2zsPNrpJZmYnzOFwEvzR/Hbuu34FR/oGuOoHT7Np\n+3uNbpKZ2QlxOJwkF5xzJvevXsFAwKo1T/Pauwca3SQzs3FzOJxEH5xzBg/csILmJrFqzdNs/P2+\nRjfJzGxcHA4n2Qc6Z/LA6o+SzzXzr+94lpe27G10k8zMxuRwOAUWzZ7BAzd8lDPzLfzpHc/y0PNb\nONTrx4ua2enL4XCKLDirjQdWf5RzCnn+8h9epOs//Zy/ePBF/s+mnQwMRKObZ2Y2hIoPX5t8urq6\noru7u9HNOGEDA0H3O3v4yQtb+MeXtnLgaB/ntE/nyg/P4/Mfmcd57zuj0U00sylM0vMR0TXmdg6H\nxjlyrJ8nX97GT17YwlOv76R/ILhofjv/6iPz+eMPncNZM1ob3UQzm2IcDpPM9gNHWLvu9zzyf3vY\n+Pv9tDSJj35gFn9w9hl8oHMmiztn8oHOGZw1o5XiYy/MzE6cw2ESe/Xd/TzyQg+//t0O3tp5kKNl\nT5krtOVYPHvGkMBY3DmTOWdOY+a0FgeHmY3K4TBFDAwEPXsP88aO93hzx8Eh79sPHB2yba5ZtOdb\nKbTl6GjLUWhrHXwvlrXSns+Rb20mn0uv1qHv03PNNDc5YMymqvGGw0SeBGenQFOTWHBWGwvOauOS\n84euO3DkGG/uOMibO99j54Fe9hzqZc+hY+w91MveQ8fYvPsQ67ccY8+h3iFnH2NpbWkin2tmWksT\nueYmWluayDUrvaey5uNlLc1N5JpEc1MTLU2iuVm0NImWpiZamkVzU/Fzc5NolmiqXBY0N5WWy9+h\nSSp7gdJ7k4r7UFoe9p7+dsWcO15HApHeq9RR+o7i8vGypnRGNnR92b6KXzPkc+V2pUdjVa6nYl+U\nlRV3q7JlBs8OVb6Nzxitzk6bcJC0EvhvQDPww4j4boObdNo7Y3qODy0o8KEFhTG3Pdzbz97Dvew7\nfIzDvf0cPtbPkWP9HOrt53Bvcflw6fOxYtmx/gGO9g1wrD841jfAsf4BevuL74d6+4rlqax/IOjr\nj+L7QNA3MEB/f3G5P332FbunRnmQFD9XCROGblRtXeV+KvdVWb9ym3Hvo2y7yq0rM2/kOmXHNGa9\nkYN0WL3ysK548GW13VTbc+X3jXCo464P8PVLl/DHHzqnytb1c1qEg6Rm4L8Dnwa2AL+VtDYiXm5s\ny6aOfGsz+dY8c9vzDWvDwEDQH8WwGCi9DzCsrLQcUVwXEQwEDERx+4GI9IL+gQCKyxHH10Xl5/T9\nEcWHm5f2WVm3tK643fH9BMX1lJfD0O0oFgyWly+nv0GpG7dUj7L1pR7e0v6oWoeq9Yv1jn8o365y\nn9XWMWzd0H2PWK9im3JR0ZZqdUf6jur7rV5n+D5ilHXjr8eo9YYfdLU/w7B2jrKPqn/GEf627flc\n9RV1dFqEA7Ac2BQRbwJIuh+4AnA4TCFNTaIJkWtudEvMbCynyx3S84DNZZ+3pDIzM2uA0yUcxkXS\nakndkrp37NjR6OaYmU1Zp0s49AALyj7PT2VDRMSaiOiKiK7Ozs5T1jgzs6w5XcLht8ASSedKagVW\nAWsb3CYzs8w6LQakI6JP0r8BfkbxUta7ImJjg5tlZpZZp0U4AETEY8BjjW6HmZmdPt1KZmZ2GnE4\nmJnZMJN24j1JO4B3aqw+G9hZx+acDqbaMU2144Gpd0xT7Xhg6h1TteN5f0SMebnnpA2HiZDUPZ5Z\nCSeTqXZMU+14YOod01Q7Hph6xzSR43G3kpmZDeNwMDOzYbIaDmsa3YCTYKod01Q7Hph6xzTVjgem\n3jHVfDyZHHMwM7PRZfXMwczMRpG5cJC0UtJrkjZJ+maj2zNRkt6WtF7SOkmT8qHaku6StF3ShrKy\nsyQ9Ken19N7RyDaeiBGO5zuSetLvtE7SZxvZxhMlaYGkX0l6WdJGSV9P5ZPydxrleCbt7yRpuqTn\nJL2Yjuk/pvKafqNMdSulJ879jrInzgFXT+Ynzkl6G+iKiEl7bbakTwDvAfdGxNJU9jfA7oj4bgrx\njoj4941s53iNcDzfAd6LiP/SyLbVStJcYG5EvCDpDOB54Ergy0zC32mU47mKSfo7qfg80RkR8Z6k\nHPAb4OvA56nhN8ramcPgE+ciohcoPXHOGigingJ2VxRfAdyTlu+h+A93UhjheCa1iNgaES+k5QPA\nKxQfyDUpf6dRjmfSiqL30sdcegU1/kZZC4ep+MS5AH4u6XlJqxvdmDqaExFb0/K7wJxGNqZOvibp\npdTtNCm6X6qRtAj4MPAsU+B3qjgemMS/k6RmSeuA7cCTEVHzb5S1cJiKPhYRy4DLgZtSl8aUEsW+\nz8ne/3k7sBhYBmwFvt/Y5tRG0kzgYeAbEbG/fN1k/J2qHM+k/p0ioj/992A+sFzS0or14/6NshYO\n43ri3GQSET3pfTvwCMWus6lgW+oXLvUPb29weyYkIralf7gDwB1Mwt8p9WM/DPw4In6Siift71Tt\neKbC7wQQEXuBXwErqfE3ylo4TKknzkmakQbTkDQD+AywYfRak8Za4Nq0fC3waAPbMmGlf5zJ55hk\nv1Ma7LwTeCUi/rZs1aT8nUY6nsn8O0nqlFRIy3mKF968So2/UaauVgJIl6b9V44/ce7mBjepZpIW\nUzxbgOKDm/5+Mh6PpPuASyjOILkN+DbwU+BBYCHF2XeviohJMcg7wvFcQrGrIoC3gRvK+oFPe5I+\nBvxvYD0wkIr/imI//aT7nUY5nquZpL+TpIsoDjg3U/wf/wcj4q8lzaKG3yhz4WBmZmPLWreSmZmN\ng8PBzMyGcTiYmdkwDgczMxvG4WBmZsM4HMzMbBiHg5mZDeNwMDOzYf4/jxtQ45b7b4gAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe1a56183c8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "def showPlot(points, title):\n",
    "    plt.figure()\n",
    "    fig, ax = plt.subplots()\n",
    "    plt.plot(points)\n",
    "\n",
    "showPlot(cbow_losses, 'CBOW Losses')\n",
    "showPlot(sg_losses, 'SkipGram Losses')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}