main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import io, os, sys, types\n",
    "from IPython import get_ipython\n",
    "from nbformat import read\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "import glob\n",
    "import traceback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_notebook(fullname, path=None):\n",
    "    \"\"\"find a notebook, given its fully qualified name and an optional path\n",
    "\n",
    "    This turns \"foo.bar\" into \"foo/bar.ipynb\"\n",
    "    and tries turning \"Foo_Bar\" into \"Foo Bar\" if Foo_Bar\n",
    "    does not exist.\n",
    "    \"\"\"\n",
    "    name = fullname.rsplit('.', 1)[-1]\n",
    "    if not path:\n",
    "        path = ['']\n",
    "    for d in path:\n",
    "        nb_path = os.path.join(d, name + \".ipynb\")\n",
    "        if os.path.isfile(nb_path):\n",
    "            return nb_path\n",
    "        # let import Notebook_Name find \"Notebook Name.ipynb\"\n",
    "        nb_path = nb_path.replace(\"_\", \" \")\n",
    "        if os.path.isfile(nb_path):\n",
    "            return nb_path\n",
    "\n",
    "\n",
    "class NotebookLoader(object):\n",
    "    \"\"\"Module Loader for Jupyter Notebooks\"\"\"\n",
    "    def __init__(self, path=None):\n",
    "        self.shell = InteractiveShell.instance()\n",
    "        self.path = path\n",
    "        \n",
    "    def load_module(self, fullname):\n",
    "        \"\"\"import a notebook as a module\"\"\"\n",
    "        path = find_notebook(fullname, self.path)\n",
    "\n",
    "        print (\"importing Jupyter notebook from %s\" % path)\n",
    "\n",
    "        # load the notebook object\n",
    "        with io.open(path, 'r', encoding='utf-8') as f:\n",
    "            nb = read(f, 4)\n",
    "\n",
    "\n",
    "        # create the module and add it to sys.modules\n",
    "        # if name in sys.modules:\n",
    "        #    return sys.modules[name]\n",
    "        mod = types.ModuleType(fullname)\n",
    "        mod.__file__ = path\n",
    "        mod.__loader__ = self\n",
    "        mod.__dict__['get_ipython'] = get_ipython\n",
    "        sys.modules[fullname] = mod\n",
    "\n",
    "        # extra work to ensure that magics that would affect the user_ns\n",
    "        # actually affect the notebook module's ns\n",
    "        save_user_ns = self.shell.user_ns\n",
    "        self.shell.user_ns = mod.__dict__\n",
    "\n",
    "        try:\n",
    "            for cell in nb.cells:\n",
    "                if cell.cell_type == 'code':\n",
    "                    # transform the input to executable Python\n",
    "                    code = self.shell.input_transformer_manager.transform_cell(cell.source)\n",
    "                    # run the code in themodule\n",
    "                    exec(code, mod.__dict__)\n",
    "        finally:\n",
    "            self.shell.user_ns = save_user_ns\n",
    "        return mod\n",
    "\n",
    "class NotebookFinder(object):\n",
    "    \"\"\"Module finder that locates Jupyter Notebooks\"\"\"\n",
    "    def __init__(self):\n",
    "        self.loaders = {}\n",
    "\n",
    "    def find_module(self, fullname, path=None):\n",
    "        nb_path = find_notebook(fullname, path)\n",
    "        if not nb_path:\n",
    "            return\n",
    "\n",
    "        key = path\n",
    "        if path:\n",
    "            # lists aren't hashable\n",
    "            key = os.path.sep.join(path)\n",
    "\n",
    "        if key not in self.loaders:\n",
    "            self.loaders[key] = NotebookLoader(path)\n",
    "        return self.loaders[key]\n",
    "\n",
    "sys.meta_path.append(NotebookFinder())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "scrapers = [state.split('.')[0] for state in glob.glob('*county.ipynb')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(scrapers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from HI by county.ipynb\n",
      "importing Jupyter notebook from NJ by county.ipynb\n",
      "importing Jupyter notebook from GA by county.ipynb\n",
      "importing Jupyter notebook from CA by county.ipynb\n",
      "importing Jupyter notebook from WA by county.ipynb\n",
      "importing Jupyter notebook from TX by county.ipynb\n",
      "Error scraping TX by county.\n",
      "importing Jupyter notebook from NY by county.ipynb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"/Users/ckirby/.pyenv/versions/3.7.2/envs/ipython/lib/python3.7/site-packages/pandas/core/internals/construction.py\", line 500, in _list_to_arrays\n",
      "    content, columns, dtype=dtype, coerce_float=coerce_float\n",
      "  File \"/Users/ckirby/.pyenv/versions/3.7.2/envs/ipython/lib/python3.7/site-packages/pandas/core/internals/construction.py\", line 583, in _convert_object_array\n",
      "    \"{con} columns\".format(col=len(columns), con=len(content))\n",
      "AssertionError: 2 columns passed, passed data had 3 columns\n",
      "\n",
      "The above exception was the direct cause of the following exception:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"<ipython-input-7-a6fb3440f6a4>\", line 6, in <module>\n",
      "    i.run()\n",
      "  File \"<string>\", line 2, in run\n",
      "  File \"<string>\", line 17, in fetch\n",
      "  File \"/Users/ckirby/.pyenv/versions/3.7.2/envs/ipython/lib/python3.7/site-packages/pandas/core/frame.py\", line 450, in __init__\n",
      "    arrays, columns = to_arrays(data, columns, dtype=dtype)\n",
      "  File \"/Users/ckirby/.pyenv/versions/3.7.2/envs/ipython/lib/python3.7/site-packages/pandas/core/internals/construction.py\", line 464, in to_arrays\n",
      "    return _list_to_arrays(data, columns, coerce_float=coerce_float, dtype=dtype)\n",
      "  File \"/Users/ckirby/.pyenv/versions/3.7.2/envs/ipython/lib/python3.7/site-packages/pandas/core/internals/construction.py\", line 503, in _list_to_arrays\n",
      "    raise ValueError(e) from e\n",
      "ValueError: 2 columns passed, passed data had 3 columns\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from AK by county.ipynb\n",
      "Error scraping AK by county.\n",
      "importing Jupyter notebook from CO by county.ipynb\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"<ipython-input-7-a6fb3440f6a4>\", line 6, in <module>\n",
      "    i.run()\n",
      "  File \"<string>\", line 2, in run\n",
      "  File \"<string>\", line 10, in fetch\n",
      "  File \"/Users/ckirby/.pyenv/versions/3.7.2/envs/ipython/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py\", line 430, in find_element_by_css_selector\n",
      "    return self.find_element(by=By.CSS_SELECTOR, value=css_selector)\n",
      "  File \"/Users/ckirby/.pyenv/versions/3.7.2/envs/ipython/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py\", line 659, in find_element\n",
      "    {\"using\": by, \"value\": value})['value']\n",
      "  File \"/Users/ckirby/.pyenv/versions/3.7.2/envs/ipython/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py\", line 633, in _execute\n",
      "    return self._parent.execute(command, params)\n",
      "  File \"/Users/ckirby/.pyenv/versions/3.7.2/envs/ipython/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py\", line 321, in execute\n",
      "    self.error_handler.check_response(response)\n",
      "  File \"/Users/ckirby/.pyenv/versions/3.7.2/envs/ipython/lib/python3.7/site-packages/selenium/webdriver/remote/errorhandler.py\", line 242, in check_response\n",
      "    raise exception_class(message, screen, stacktrace)\n",
      "selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {\"method\":\"css selector\",\"selector\":\"th\"}\n",
      "  (Session info: chrome=80.0.3987.149)\n",
      "  (Driver info: chromedriver=2.42.591059 (a3d9684d10d61aa0c45f6723b327283be1ebaad8),platform=Mac OS X 10.14.6 x86_64)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from MI by county.ipynb\n",
      "importing Jupyter notebook from VA by county.ipynb\n",
      "importing Jupyter notebook from AL by county.ipynb\n",
      "importing Jupyter notebook from FL by county.ipynb\n",
      "importing Jupyter notebook from OH by county.ipynb\n",
      "importing Jupyter notebook from MD by county.ipynb\n",
      "importing Jupyter notebook from PA by county.ipynb\n",
      "importing Jupyter notebook from TN by county.ipynb\n",
      "importing Jupyter notebook from WY by county.ipynb\n"
     ]
    }
   ],
   "source": [
    "import importlib\n",
    "\n",
    "for scraper in scrapers:\n",
    "    i = importlib.import_module(scraper)\n",
    "    try:\n",
    "        i.run()\n",
    "    except:\n",
    "        print(f'Error scraping {scraper}.')\n",
    "        traceback.print_exc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "importing Jupyter notebook from LA by county.ipynb\n"
     ]
    }
   ],
   "source": [
    "import LA_by_county as c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "c.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}