NJ by county.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Maryland updates their data daily at 10am EDT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import time\n",
    "import pandas as pd\n",
    "import pendulum\n",
    "import re\n",
    "import yaml\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "chrome_options = Options()\n",
    "#chrome_options.add_argument(\"--disable-extensions\")\n",
    "#chrome_options.add_argument(\"--disable-gpu\")\n",
    "#chrome_options.add_argument(\"--no-sandbox) # linux only\n",
    "chrome_options.add_argument(\"--start-maximized\")\n",
    "# chrome_options.add_argument(\"--headless\")\n",
    "chrome_options.add_argument(\"user-agent=[Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:73.0) Gecko/20100101 Firefox/73.0]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('config.yaml', 'r') as f:\n",
    "    config = yaml.safe_load(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "state = 'NJ'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "scrape_timestamp = pendulum.now().strftime('%Y%m%d%H%M%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MD positive cases by county\n",
    "url = 'https://maps.arcgis.com/apps/opsdashboard/index.html#/ec4bffd48f7e495182226eee7962b422'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch():\n",
    "    driver = webdriver.Chrome('../20190611 - Parts recommendation/chromedriver', options=chrome_options)\n",
    "\n",
    "    driver.get(url)\n",
    "    time.sleep(5)\n",
    "\n",
    "    # class topBoxH1Text is used in the summary box.\n",
    "    datatbl = driver.find_element_by_id('ember13')\n",
    "    datatbl = datatbl.find_elements_by_class_name('external-html')\n",
    "\n",
    "    data = [re.search('^(.*):     \\n(\\d*)', row.text) for row in datatbl]\n",
    "\n",
    "    data = [row.groups() for row in data if row]\n",
    "    page_source = driver.page_source\n",
    "    driver.close()\n",
    "\n",
    "    return pd.DataFrame(data, columns=['county','positive_cases']), page_source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save(df, source):\n",
    "    df.to_csv(f\"{config['data_folder']}/{state}_county_{scrape_timestamp}.txt\", sep='|', index=False)\n",
    "\n",
    "    with open(f\"{config['data_source_backup_folder']}/{state}_county_{scrape_timestamp}.html\", 'w') as f:\n",
    "        f.write(source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run():\n",
    "    df, source = fetch()\n",
    "    save(df, source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}