nn-hindi.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Hindi ULMFiT from scratch based on Rachel/Jeremy's awesome [Code-First Intro to Natural Language Processing](https://www.youtube.com/playlist?list=PL8l5P33wvCvIqO3UyvdkeRMoGg--VBvho) and its [Notebook](https://github.com/fastai/course-nlp/blob/master/nn-vietnamese.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "from fastai import *\n",
    "from fastai.text import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# bs=48\n",
    "# bs=24\n",
    "bs=128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.set_device(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ash022/.fastai/data\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/fastai/datasets.py:164: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  with open(fpath, 'r') as yaml_file: return yaml.load(yaml_file)\n"
     ]
    }
   ],
   "source": [
    "data_path = Config.data_path()\n",
    "print(data_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This will create a `viwiki` folder, containing a `viwiki` text file with the wikipedia contents. (For other languages, replace `vi` with the appropriate code from the [list of wikipedias](https://meta.wikimedia.org/wiki/List_of_Wikipedias).)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "lang = 'hi'\n",
    "#lang = 'vi'\n",
    "# lang = 'zh'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = f'{lang}wiki'\n",
    "path = data_path/name\n",
    "path.mkdir(exist_ok=True, parents=True)\n",
    "lm_fns = [f'{lang}_wt', f'{lang}_wt_vocab']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Hindi wikipedia model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nlputils import split_wiki,get_wiki"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unzipping...\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for /: 'str' and 'str'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-aa2f9a5ff8b6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_wiki\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlang\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/mnt/promec-ns9036k/NORSTORE_OSL_DISK/NS9036K/promec/promec/Animesh/notebooks/nlputils.py\u001b[0m in \u001b[0;36mget_wiki\u001b[0;34m(path, lang)\u001b[0m\n\u001b[1;32m     16\u001b[0m         \u001b[0mdownload_url\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'https://dumps.wikimedia.org/{name}/latest/{zip_fn}'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mzip_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"unzipping...\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m         \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'bunzip2 %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mpath\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mzip_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m         \u001b[0;31m#bunzip2(path/zip_fn)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mzip_fn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for /: 'str' and 'str'"
     ]
    }
   ],
   "source": [
    "get_wiki(path,lang)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "!bunzip2 /home/ash022/.fastai/data/hiwiki/hiwiki-latest-pages-articles.xml.bz2\n",
    "!git clone https://github.com/fonnesbeck/Bios8366.git\n",
    "!python WikiExtractor.py --processes 4 --no_templates  --min_text_length 1800 --filter_disambig_pages --log_file log -b 100G -q $HOME/.fastai/data/hiwiki/hiwiki-latest-pages-articles.xml\n",
    "!mv text/AA/wiki_00 $HOME/.fastai/data/hiwiki/hiwiki"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "name=path.ls()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<mediawiki xmlns=\"http://www.mediawiki.org/xml/export-0.10/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\" xsi:schemaLocation=\"http://www.mediawiki.org/xml/export-0.10/ http://www.mediawiki.org/xml/export-0.10.xsd\" version=\"0.10\" xml:lang=\"hi\">\n",
      "  <siteinfo>\n",
      "    <sitename>विकिपीडिया</sitename>\n",
      "    <dbname>hiwiki</dbname>\n"
     ]
    }
   ],
   "source": [
    "!head -n4 /home/ash022/.fastai/data/hiwiki/hiwiki-latest-pages-articles.xml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This function splits the single wikipedia file into a separate file per article. This is often easier to work with."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ash022/.fastai/data/hiwiki/docs already exists; not splitting\n"
     ]
    }
   ],
   "source": [
    "dest = split_wiki(path,lang)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PosixPath('/home/ash022/.fastai/data/hiwiki/docs/पाड़.txt'),\n",
       " PosixPath('/home/ash022/.fastai/data/hiwiki/docs/इलाहाबाद.txt'),\n",
       " PosixPath('/home/ash022/.fastai/data/hiwiki/docs/सैयद अता हसनैन.txt'),\n",
       " PosixPath('/home/ash022/.fastai/data/hiwiki/docs/रविन्द्र चौबे.txt'),\n",
       " PosixPath('/home/ash022/.fastai/data/hiwiki/docs/मैरी कॉम.txt')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dest.ls()[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use this to convert Chinese traditional to simplified characters\n",
    "# ls *.txt | parallel -I% opencc -i % -o ../zhsdocs/% -c t2s.json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create pretrained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60004, 15147)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = (TextList.from_folder(dest)\n",
    "            .split_by_rand_pct(0.1, seed=42)\n",
    "            .label_for_lm()           \n",
    "            .databunch(bs=bs, num_workers=1))\n",
    "\n",
    "data.save(f'{lang}_databunch')\n",
    "len(data.vocab.itos),len(data.train_ds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/ash022/.fastai/data/hiwiki hi_databunch\n"
     ]
    }
   ],
   "source": [
    "print(path,f'{lang}_databunch')\n",
    "!mv /home/ash022/.fastai/data/hiwiki/docs/hi_databunch /home/ash022/.fastai/data/hiwiki/hi_databunch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "language_model_learner() missing 2 required positional arguments: 'data' and 'arch'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-35-ba3f9448f41a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m#data = load_data(path, 'docs', bs=bs)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mempty_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mlanguage_model_learner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdestroy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: language_model_learner() missing 2 required positional arguments: 'data' and 'arch'"
     ]
    }
   ],
   "source": [
    "#data = load_data(path, f'{lang}_databunch', bs=bs)\n",
    "#data = load_data(path, 'docs', bs=bs)\n",
    "torch.cuda.empty_cache()\n",
    "language_model_learner()\n",
    "learn.destroy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n",
      "[2. 4. 6.]\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-48f81c660e9e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mselect_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m \u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/gpu:1'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/numba/cuda/api.py\u001b[0m in \u001b[0;36mselect_device\u001b[0;34m(device_id)\u001b[0m\n\u001b[1;32m    300\u001b[0m     \u001b[0mRaises\u001b[0m \u001b[0mexception\u001b[0m \u001b[0mon\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m     \"\"\"\n\u001b[0;32m--> 302\u001b[0;31m     \u001b[0mcontext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdevices\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    303\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    304\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/numba/cuda/cudadrv/devices.py\u001b[0m in \u001b[0;36mget_context\u001b[0;34m(devnum)\u001b[0m\n\u001b[1;32m    192\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mCUDA\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m     \"\"\"\n\u001b[0;32m--> 194\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_runtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_or_create_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevnum\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/numba/cuda/cudadrv/devices.py\u001b[0m in \u001b[0;36mget_or_create_context\u001b[0;34m(self, devnum)\u001b[0m\n\u001b[1;32m    160\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 162\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpush_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgpus\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdevnum\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    163\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/numba/cuda/cudadrv/devices.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, devnum)\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0mReturns\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mcontext\u001b[0m \u001b[0mmanager\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mdevice\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mdevnum\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m         '''\n\u001b[0;32m---> 40\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlst\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdevnum\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__str__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from numba import cuda\n",
    "\n",
    "a = tf.constant([1.0,2.0,3.0],shape=[3],name='a')\n",
    "b = tf.constant([1.0,2.0,3.0],shape=[3],name='b')\n",
    "with tf.device('/gpu:1'):\n",
    "    c = a+b\n",
    "\n",
    "TF_CONFIG = tf.ConfigProto(\n",
    "gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.1),\n",
    "  allow_soft_placement=True)\n",
    "\n",
    "sess = tf.Session(config=TF_CONFIG)\n",
    "sess.run(tf.global_variables_initializer())\n",
    "i=1\n",
    "while(i<1000):\n",
    "        i=i+1\n",
    "        print(sess.run(c))\n",
    "\n",
    "sess.close()\n",
    "cuda.select_device(1)\n",
    "cuda.close()\n",
    "with tf.device('/gpu:1'):\n",
    "    c = a+b\n",
    "\n",
    "TF_CONFIG = tf.ConfigProto(\n",
    "gpu_options=tf.GPUOptions(per_process_gpu_memory_fraction=0.5),\n",
    "  allow_soft_placement=True)\n",
    "\n",
    "sess = tf.Session(config=TF_CONFIG)\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "while(1):\n",
    "        print(sess.run(c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "learn = language_model_learner(data, AWD_LSTM, drop_mult=0.5, pretrained=False)#.to_fp16()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1e-2\n",
    "lr *= bs/48  # Scale learning rate by batch size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='10', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/10 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='21' class='' max='2486', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.84% [21/2486 00:06<12:28 8.2070]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(10, lr, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the pretrained model and vocab:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "mdl_path = path/'models'\n",
    "mdl_path.mkdir(exist_ok=True)\n",
    "learn.to_fp32().save(mdl_path/lm_fns[0], with_opt=False)\n",
    "learn.data.vocab.save(mdl_path/(lm_fns[1] + '.pkl'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sentiment analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Language model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- [Data](https://github.com/shubham721/Sentiment-Analysis-On-Hindi-Reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:1: FutureWarning: read_table is deprecated, use read_csv instead, passing sep='\\t'.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>मिकी वाइरस पूरी तरह से मनीष पॉल की फिल्म है और...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>$मनीष ने अपने किरकार को बखूबी निभाया है</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>$उन्होंने शाहिद के डर, खौफ, संघर्ष, जिद और जीत...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>$वे हर भाव के दृश्यों में नैचुरल लगते हैं</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>$फिल्म की कास्टिंग जबरदस्त है</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             comment  label\n",
       "0  मिकी वाइरस पूरी तरह से मनीष पॉल की फिल्म है और...      1\n",
       "1            $मनीष ने अपने किरकार को बखूबी निभाया है      1\n",
       "2  $उन्होंने शाहिद के डर, खौफ, संघर्ष, जिद और जीत...      1\n",
       "3          $वे हर भाव के दृश्यों में नैचुरल लगते हैं      1\n",
       "4                      $फिल्म की कास्टिंग जबरदस्त है      1"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_df = pd.read_table('pos_hindi.txt',header=None)\n",
    "pos_df.columns=['comment']\n",
    "pos_df['label']=1\n",
    "#train_df.loc[pd.isna(train_df.comment),'comment']='NA'\n",
    "pos_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/ipykernel_launcher.py:1: FutureWarning: read_table is deprecated, use read_csv instead.\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>फिल्म देखने के बाद न सिर्फ उम्मीदें धराशायी हु...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>$उनके अभिनय में इंटेसिंटी तो है लेकिन किरदार स...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>$एली अबराम को फिल्म में महज ग्लैमर ऐड करने के ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>$ बाकी के भी किरदार अति सामान्य हैं</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>$ संदर्भ और स्थितियां बदल गई हैं, लेकिन सोच और...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             comment  label\n",
       "0  फिल्म देखने के बाद न सिर्फ उम्मीदें धराशायी हु...      0\n",
       "1  $उनके अभिनय में इंटेसिंटी तो है लेकिन किरदार स...      0\n",
       "2  $एली अबराम को फिल्म में महज ग्लैमर ऐड करने के ...      0\n",
       "3                $ बाकी के भी किरदार अति सामान्य हैं      0\n",
       "4  $ संदर्भ और स्थितियां बदल गई हैं, लेकिन सोच और...      0"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neg_df = pd.read_table('neg_hindi.txt',header=None,sep=\"\\n\")\n",
    "neg_df.columns=['comment']\n",
    "neg_df['label']=0\n",
    "#test_df.loc[pd.isna(test_df.comment),'comment']='NA'\n",
    "neg_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(                                             comment  label\n",
       " 0  मिकी वाइरस पूरी तरह से मनीष पॉल की फिल्म है और...      1\n",
       " 1            $मनीष ने अपने किरकार को बखूबी निभाया है      1\n",
       " 2  $उन्होंने शाहिद के डर, खौफ, संघर्ष, जिद और जीत...      1\n",
       " 3          $वे हर भाव के दृश्यों में नैचुरल लगते हैं      1\n",
       " 4                      $फिल्म की कास्टिंग जबरदस्त है      1,\n",
       "                                                comment  label\n",
       " 506  $अगर डायरेक्टर बार - बार हीरो राम द्वारा घर की...      0\n",
       " 507     $मैं गलियों से निकलता हूं फिर भी फंस जाता हूं।      0\n",
       " 508     $क्लाइमेक्स भी बेवजह खींचा गया महसूस होता है।       0\n",
       " 509  $हम इस धारणा के प्रभाव को महसूस करते हैं, लेकि...      0\n",
       " 510                                                  $      0)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.concat([pos_df,neg_df], sort=False)\n",
    "df.head(), df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lm = (TextList.from_df(df, path, cols='comment')\n",
    "    .split_by_rand_pct(0.1, seed=42)\n",
    "    .label_for_lm()           \n",
    "    .databunch(bs=bs, num_workers=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/fastai/datasets.py:164: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  with open(fpath, 'r') as yaml_file: return yaml.load(yaml_file)\n"
     ]
    }
   ],
   "source": [
    "learn_lm = language_model_learner(data_lm, AWD_LSTM, pretrained_fnames=lm_fns, drop_mult=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1e-3\n",
    "lr *= bs/48"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 00:00 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>5.137067</td>\n",
       "      <td>4.378086</td>\n",
       "      <td>0.276228</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4.962988</td>\n",
       "      <td>4.295090</td>\n",
       "      <td>0.279464</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_lm.fit_one_cycle(2, lr*10, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Total time: 00:02 <p><table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.638175</td>\n",
       "      <td>4.205736</td>\n",
       "      <td>0.272768</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4.601702</td>\n",
       "      <td>4.058837</td>\n",
       "      <td>0.273326</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>4.526342</td>\n",
       "      <td>3.834914</td>\n",
       "      <td>0.288170</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.425710</td>\n",
       "      <td>3.666122</td>\n",
       "      <td>0.305580</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4.316147</td>\n",
       "      <td>3.554887</td>\n",
       "      <td>0.315290</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>4.215147</td>\n",
       "      <td>3.546034</td>\n",
       "      <td>0.326004</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>4.131269</td>\n",
       "      <td>3.513413</td>\n",
       "      <td>0.331920</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>4.072524</td>\n",
       "      <td>3.486128</td>\n",
       "      <td>0.333147</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_lm.unfreeze()\n",
    "learn_lm.fit_one_cycle(8, lr, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_lm.save(f'{lang}fine_tuned')\n",
    "learn_lm.save_encoder(f'{lang}fine_tuned_enc')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas = (TextList.from_df(pos_df, path, vocab=data_lm.vocab, cols='comment')\n",
    "    .split_by_rand_pct(0.1, seed=42)\n",
    "    .label_from_df(cols='label')\n",
    "    .databunch(bs=bs, num_workers=1))\n",
    "\n",
    "data_clas.save(f'{lang}_textlist_class')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas = load_data(path, f'{lang}_textlist_class', bs=bs, num_workers=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "import numpy as np\n",
    "#@np_func\n",
    "def f1(inp,targ): return f1_score(targ, np.argmax(inp, axis=-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/fastai/datasets.py:164: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n",
      "  with open(fpath, 'r') as yaml_file: return yaml.load(yaml_file)\n"
     ]
    }
   ],
   "source": [
    "learn_c = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5, metrics=[accuracy,f1]).to_fp16()\n",
    "learn_c.load_encoder(f'{lang}fine_tuned_enc')\n",
    "learn_c.freeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=2e-2\n",
    "lr *= bs/48"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='2', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/2 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>f1</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='progress-bar-interrupted' max='3', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      Interrupted\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "ename": "RuntimeError",
     "evalue": "output with type torch.cuda.HalfTensor doesn't match the desired type torch.cuda.FloatTensor",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-72-930d44b01fce>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearn_c\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_one_cycle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmoms\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0.8\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0.7\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_fp16\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/fastai/train.py\u001b[0m in \u001b[0;36mfit_one_cycle\u001b[0;34m(learn, cyc_len, max_lr, moms, div_factor, pct_start, final_div, wd, callbacks, tot_epochs, start_epoch)\u001b[0m\n\u001b[1;32m     20\u001b[0m     callbacks.append(OneCycleScheduler(learn, max_lr, moms=moms, div_factor=div_factor, pct_start=pct_start,\n\u001b[1;32m     21\u001b[0m                                        final_div=final_div, tot_epochs=tot_epochs, start_epoch=start_epoch))\n\u001b[0;32m---> 22\u001b[0;31m     \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcyc_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_lr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mlr_find\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mLearner\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_lr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1e-7\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_lr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_it\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstop_div\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[1;32m    194\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallback_fns\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callbacks\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mdefaults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mextra_callbacks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m         \u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m->\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[1;32m     98\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0myb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 100\u001b[0;31m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0myb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    101\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    102\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/fastai/basic_train.py\u001b[0m in \u001b[0;36mloss_batch\u001b[0;34m(model, xb, yb, loss_func, opt, cb_handler)\u001b[0m\n\u001b[1;32m     23\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mxb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mis_listy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0myb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0myb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0myb\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m     \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mxb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m     \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_loss_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/torch/nn/modules/container.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m     90\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_modules\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodule\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    487\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 489\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    490\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.6/site-packages/fastai/text/learner.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    234\u001b[0m         \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    235\u001b[0m         \u001b[0mavg_pool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmasked_fill\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 236\u001b[0;31m         \u001b[0mavg_pool\u001b[0m \u001b[0;34m*=\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    237\u001b[0m         \u001b[0mmax_pool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutput\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmasked_fill\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'inf'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    238\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_pool\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mavg_pool\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: output with type torch.cuda.HalfTensor doesn't match the desired type torch.cuda.FloatTensor"
     ]
    }
   ],
   "source": [
    "learn_c.fit_one_cycle(2, lr, moms=(0.8,0.7)).to_fp16()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>_inner</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.321768</td>\n",
       "      <td>0.255457</td>\n",
       "      <td>0.899254</td>\n",
       "      <td>0.871367</td>\n",
       "      <td>00:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.305934</td>\n",
       "      <td>0.250888</td>\n",
       "      <td>0.894901</td>\n",
       "      <td>0.872021</td>\n",
       "      <td>00:02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_c.fit_one_cycle(2, lr, moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>_inner</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.300939</td>\n",
       "      <td>0.261080</td>\n",
       "      <td>0.893657</td>\n",
       "      <td>0.866201</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.263790</td>\n",
       "      <td>0.220207</td>\n",
       "      <td>0.906716</td>\n",
       "      <td>0.886115</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_c.freeze_to(-2)\n",
    "learn_c.fit_one_cycle(2, slice(lr/(2.6**4),lr), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>_inner</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.282888</td>\n",
       "      <td>0.238203</td>\n",
       "      <td>0.905473</td>\n",
       "      <td>0.886483</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.248599</td>\n",
       "      <td>0.216489</td>\n",
       "      <td>0.918532</td>\n",
       "      <td>0.901550</td>\n",
       "      <td>00:04</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_c.freeze_to(-3)\n",
    "learn_c.fit_one_cycle(2, slice(lr/2/(2.6**4),lr/2), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>_inner</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.201508</td>\n",
       "      <td>0.217176</td>\n",
       "      <td>0.911070</td>\n",
       "      <td>0.890084</td>\n",
       "      <td>00:05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn_c.unfreeze()\n",
    "learn_c.fit_one_cycle(1, slice(lr/10/(2.6**4),lr/10), moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn_c.save(f'{lang}clas')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Competition top 3 f1 scores: 0.90, 0.89, 0.89. Winner used an ensemble of 4 models: TextCNN, VDCNN, HARNN, and SARNN."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas = load_data(path, f'{lang}_textlist_class', bs=bs, num_workers=1)\n",
    "learn_c = text_classifier_learner(data_clas, AWD_LSTM, drop_mult=0.5, metrics=[accuracy,f1]).to_fp16()\n",
    "learn_c.load(f'{lang}clas', purge=False);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.9111), tensor(0.8952))"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds,targs = learn_c.get_preds(ordered=True)\n",
    "accuracy(preds,targs),f1(preds,targs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_clas_bwd = load_data(path, f'{lang}_textlist_class_bwd', bs=bs, num_workers=1, backwards=True)\n",
    "learn_c_bwd = text_classifier_learner(data_clas_bwd, AWD_LSTM, drop_mult=0.5, metrics=[accuracy,f1]).to_fp16()\n",
    "learn_c_bwd.load(f'{lang}clas_bwd', purge=False);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.9092), tensor(0.8957))"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_b,targs_b = learn_c_bwd.get_preds(ordered=True)\n",
    "accuracy(preds_b,targs_b),f1(preds_b,targs_b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_avg = (preds+preds_b)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.9154), tensor(0.9016))"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy(preds_avg,targs_b),f1(preds_avg,targs_b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Future work\n",
    "* Try [SentencePiece](https://github.com/google/sentencepiece)\n",
    "* Check activation with [Interpretability Beyond Feature Attribution: Quantitative Testing with Concept Activation Vectors](https://github.com/tensorflow/tcav)\n",
    "* [GPT2](https://gist.github.com/mohdsanadzakirizvi/f1419fba3907af2baf6e0f6ab2a53d5b)\n",
    "* [Unsupervised word embeddings capture latent knowledge from materials science literature](https://www.nature.com/articles/s41586-019-1335-8)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}