hatebase_graph.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hatebase_lexicon = '../data/hatebase/lexicon.csv'\n",
    "hatebase_fields = {\n",
    "    'about_class': np.int32, \n",
    "    'about_religion': np.int32, \n",
    "    'about_gender': np.int32,\n",
    "    'about_ethnicity': np.int32, \n",
    "    'about_nationality': np.int32, \n",
    "    'about_sexual_orientation': np.int32, \n",
    "    'about_disability': np.int32\n",
    "    # 'number_of_sightings': np.int32\n",
    "}\n",
    "with open(hatebase_lexicon,'rb') as hb:\n",
    "    hatebase_data = pd.read_csv( hb, header = 0, index_col = 0, quoting = 0, \n",
    "                                    dtype = hatebase_fields, usecols = range(8) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print hatebase_data.index[hatebase_data[field] == 1].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "graph = defaultdict(set) # dict of lists\n",
    "for field in hatebase_fields.keys():\n",
    "    field_words = hatebase_data.index[hatebase_data[field] == 1].tolist()\n",
    "    for word in field_words:\n",
    "        graph[word] |= set(field_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trimmed_graph = {}\n",
    "equiv = set([])\n",
    "for word in graph.keys():\n",
    "#     graph[word].remove(word)\n",
    "    trimmed_graph[word] = list(graph[word])\n",
    "    equiv.add(len(trimmed_graph[word]))\n",
    "    print word, len(trimmed_graph[word])\n",
    "print equiv, len(equiv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "G = nx.Graph(trimmed_graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = nx.to_pandas_dataframe(G)\n",
    "\n",
    "graph_write_path = '../data/hatebase/adjacency_matrix.csv'\n",
    "df.to_csv(graph_write_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}