LA by county.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Louisianna updates daily at noon CDT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import time\n",
    "import pandas as pd\n",
    "import pendulum\n",
    "import re\n",
    "import yaml\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "chrome_options = Options()\n",
    "#chrome_options.add_argument(\"--disable-extensions\")\n",
    "#chrome_options.add_argument(\"--disable-gpu\")\n",
    "#chrome_options.add_argument(\"--no-sandbox) # linux only\n",
    "chrome_options.add_argument(\"--start-maximized\")\n",
    "# chrome_options.add_argument(\"--headless\")\n",
    "chrome_options.add_argument(\"user-agent=[Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:73.0) Gecko/20100101 Firefox/73.0]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('config.yaml', 'r') as f:\n",
    "    config = yaml.safe_load(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "state = 'LA'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "scrape_timestamp = pendulum.now().strftime('%Y%m%d%H%M%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://www.arcgis.com/apps/opsdashboard/index.html#/69b726e2b82e408f89c3a54f96e8f776'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch():\n",
    "    driver = webdriver.Chrome('../20190611 - Parts recommendation/chromedriver', options=chrome_options)\n",
    "\n",
    "    driver.get(url)\n",
    "    time.sleep(5)\n",
    "    \n",
    "    datatbl = driver.find_element_by_id('ember105')\n",
    "    datatbl.click()\n",
    "    time.sleep(5)\n",
    "    \n",
    "    datatbl = driver.find_element_by_class_name('feature-list')\n",
    "\n",
    "    datatbl.find_elements_by_class_name('external-html')\n",
    "    datatbl = datatbl.text.split('\\n')\n",
    "    data = [d.split('-') for d in datatbl]\n",
    "    data = [[d[0]] + d[1].split('|') for d in data[:-1]]\n",
    "    \n",
    "    page_source = driver.page_source\n",
    "    driver.close()\n",
    "    return pd.DataFrame(data, columns=['county','positive_cases','deaths']), page_source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save(df, source):\n",
    "    df.to_csv(f\"{config['data_folder']}/{state}_county_{scrape_timestamp}.txt\", sep='|', index=False)\n",
    "\n",
    "    with open(f\"{config['data_source_backup_folder']}/{state}_county_{scrape_timestamp}.html\", 'w') as f:\n",
    "        f.write(source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run():\n",
    "    df, source = fetch()\n",
    "    save(df, source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}