scripts/NJCovid19Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading NJ COVID-19 Data Into JSON and CSV\n",
    "\n",
    "This python notebook contains the code used to scrape articles from [NJ.com](https://www.nj.com/) to get data on number of COVID-19 cases per municipalities in New Jersey. \n",
    "\n",
    "I use `urllib` for loading the webpage, `bs4` & `re` for scraping, `datetime` for getting the current date, and `json` & `csv` for loading the data into the correct outputting files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib import request, error, parse\n",
    "from bs4 import BeautifulSoup\n",
    "from datetime import date\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json, csv, re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Getting the current date and loading it into the correct format for accessing the article url where the data lies."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def current_date():\n",
    "    rn = date.today()\n",
    "    month = rn.strftime(\"%B\").lower()\n",
    "    month_number = str(rn.month) if rn.month > 9 else '0' + str(rn.month)\n",
    "    return \"{}-{}-{}\".format(month,rn.day,rn.year), month_number\n",
    "\n",
    "def get_date(month,day):\n",
    "    return \"{}-{}-{}\".format(month,day,2020)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading the HTML file locally for scraping and parsing. Getting the main part of the article where the data lies within the HTML file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_html(link):\n",
    "    response = request.urlopen(link)\n",
    "    covid19_html = response.read()\n",
    "    soup = BeautifulSoup(covid19_html,'lxml')\n",
    "    return soup.find_all('article')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading and cleaning the data (in string format) for matching. Prints an error if the row is not cleaned properly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_data(article):\n",
    "    data = []\n",
    "    for row in article.find_all('p'):\n",
    "        delims = '.+\\:\\s*\\d*,?\\d+'\n",
    "        statistic = row.getText().lower()\n",
    "\n",
    "        # weird april error on a particular set of days\n",
    "        if '• springfield ' in statistic:\n",
    "            statistic = statistic.replace('• springfield ','• springfield: ')\n",
    "\n",
    "        if ':' in statistic and '•' in statistic:\n",
    "            cleaned_row = re.findall(delims,statistic)\n",
    "            if len(cleaned_row) != 1:\n",
    "                print(statistic)\n",
    "            else:\n",
    "                data.append(cleaned_row[0][1:].strip())\n",
    "        \n",
    "        # weird march error on particular day\n",
    "        if '• west orange' in statistic:\n",
    "            data.append('gloucester county')\n",
    "\n",
    "        for county in counties:\n",
    "            if county in statistic:\n",
    "                data.append(county)\n",
    "                break\n",
    "    print()            \n",
    "    return data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A dictionary containing the historical typos made in data parsing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "errors = json.load(open('../data/errors.json'))\n",
    "substring_errors = errors[0]\n",
    "fullstring_errors = errors[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading the data into a json file `nj_covid19_today.json` containing the number for cases per municipalities in New Jersey for the current date."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_errors(town, fullstring_errs=fullstring_errors, substring_errs=substring_errors):\n",
    "    for error in substring_errs.keys():\n",
    "        town = town.replace(error,substring_errs[error])\n",
    "        \n",
    "    for error in fullstring_errs.keys():\n",
    "        if town.strip() == error:\n",
    "            return fullstring_errs[error]\n",
    "    \n",
    "    return town"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_town(row, county, visited_municipal):\n",
    "    infected_township = re.split(':',row)\n",
    "    town = check_errors(infected_township[0].strip()).strip()\n",
    "    num_infected = int(re.findall('^\\d+',infected_township[1].replace(',','').strip())[0])\n",
    "    \n",
    "    township = town + ' township'\n",
    "    city = town + ' city'\n",
    "    borough = town + ' borough'\n",
    "    \n",
    "    if town == 'address not reported' or town == 'unknown':\n",
    "        return 'other', num_infected\n",
    "        \n",
    "    elif town in visited_municipal and visited_municipal[town] == False:\n",
    "        visited_municipal[town] = True\n",
    "        return town,num_infected\n",
    "    \n",
    "    elif township in visited_municipal and visited_municipal[township] == False:\n",
    "        visited_municipal[township] = True\n",
    "        return township,num_infected\n",
    "    \n",
    "    elif city in visited_municipal and visited_municipal[city] == False:\n",
    "        visited_municipal[city] = True \n",
    "        return city, num_infected\n",
    "    \n",
    "    elif borough in visited_municipal and visited_municipal[borough] == False:\n",
    "        visited_municipal[borough] = True \n",
    "        return borough, num_infected\n",
    "    \n",
    "    else: \n",
    "        print('ERROR TOWN NOT FOUND IN {}: {}'.format(county,town))\n",
    "        return '',-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def put_data_in_dataframe(data, covid_df, municipals, date):\n",
    "    todays_data = []\n",
    "    current_county = ''\n",
    "\n",
    "    for row in data:\n",
    "        if row in counties:\n",
    "            current_county = row\n",
    "            continue\n",
    "\n",
    "        if current_county == '':\n",
    "            continue\n",
    "\n",
    "        municipal, infected = replace_town(row, current_county, municipals[current_county])\n",
    "        \n",
    "        if municipal == '' or infected < 0:\n",
    "            continue\n",
    "        \n",
    "        todays_data.append(\n",
    "            pd.DataFrame(\n",
    "                data=[[municipal,current_county,infected,date]],\n",
    "                columns=['Municipal','County','Cases','Date']\n",
    "            )\n",
    "        )\n",
    "\n",
    "    return covid_df.append(pd.concat(todays_data), ignore_index=True).copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EXECUTABLE\n",
    "\n",
    "Run these cells to update the data based on a certain day."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finding the correct date and link for that date's COVID-19 data in New Jersey"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# date, month = current_date()\n",
    "date, month = get_date('may',20), '05'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "nj_dot_com_link = 'https://www.nj.com/coronavirus/2020/{}/where-is-the-coronavirus\\\n",
    "-in-nj-latest-map-update-on-county-by-county-cases-{}.html'.format(month,date)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading all of the NJ Municipalities names from `nj_municipals.json`. Loading dictionaries for handling errors such as misspellings and shorthands for munipalities."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "nj_municipals = json.load(open('../data/nj_municipals.json'))\n",
    "counties = list(nj_municipals.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "article = get_html(nj_dot_com_link) # article component\n",
    "covid_data = clean_data(article[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Updating the total dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "nj_covid_df = pd.read_csv('../data/ex_nj_total.csv')\n",
    "updated_covid_df = put_data_in_dataframe(covid_data, nj_covid_df, nj_municipals, date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     Municipal         County Cases           Date\n",
      "0    allendale  bergen county     4  march-25-2020\n",
      "1       alpine  bergen county     1  march-25-2020\n",
      "2  bergenfield  bergen county    26  march-25-2020\n",
      "3       bogota  bergen county     8  march-25-2020\n",
      "4    carlstadt  bergen county     6  march-25-2020\n",
      "                 Municipal         County Cases         Date\n",
      "27658         phillipsburg  warren county   118  may-20-2020\n",
      "27659   pohatcong township  warren county    20  may-20-2020\n",
      "27660           washington  warren county    50  may-20-2020\n",
      "27661  washington township  warren county    39  may-20-2020\n",
      "27662       white township  warren county    51  may-20-2020\n"
     ]
    }
   ],
   "source": [
    "print(updated_covid_df.head())\n",
    "print(updated_covid_df.tail())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Making sure there were no leftover errors as a result of error filtering."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(updated_covid_df[updated_covid_df.Municipal == '']) == 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading the updated data into their respective locations. The new total will go to `nj_total.csv` and today's data will go to `nj_today.json`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "updated_covid_df.to_csv('../data/ex_nj_total.csv',mode='w',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "updated_covid_df = updated_covid_df[updated_covid_df['Date'] == date]\n",
    "updated_covid_df.to_json('../data/ex_nj_today.json', orient='records')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AUTOMATION\n",
    "\n",
    "Some functions I created to automate the process of data collection over a month's time period."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_link(date):\n",
    "    link = 'https://www.nj.com/coronavirus/2020/05/where-is-the-coronavirus-in-nj-latest-map-update-on-county-by-county-cases-{}.html'\n",
    "    return link.format(date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def automate(month,i,j):\n",
    "    for d in range(i,j):\n",
    "        current_date = get_date(month,d)\n",
    "        nj_link = get_link(current_date)\n",
    "        \n",
    "        print(current_date)\n",
    "        \n",
    "        municipals = json.load(open('nj_municipals.json'))\n",
    "        \n",
    "        article = get_html(nj_link) # article component\n",
    "        covid_data = clean_data(article[0])\n",
    "        \n",
    "        nj_covid_df = pd.read_csv('ddata.csv')\n",
    "        updated_covid_df = put_data_in_dataframe(covid_data, nj_covid_df, municipals, current_date)\n",
    "        \n",
    "        assert len(updated_covid_df[updated_covid_df.Municipal == '']) == 0\n",
    "        \n",
    "        updated_covid_df.to_csv('data.csv',mode='w',index=False)\n",
    "        print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weird_links(nj_link):\n",
    "    municipals = json.load(open('nj_municipals.json'))\n",
    "        \n",
    "    article = get_html(nj_link) # article component\n",
    "    covid_data = clean_data(article[0])\n",
    "\n",
    "    nj_covid_df = pd.read_csv('data.csv')\n",
    "    updated_covid_df = put_data_in_dataframe(covid_data, nj_covid_df, municipals, current_date)\n",
    "\n",
    "    assert len(updated_covid_df[updated_covid_df.Municipal == '']) == 0\n",
    "\n",
    "    updated_covid_df.to_csv('data.csv',mode='w',index=False)\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# automate('april',1,31)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}