Clean_and_Plot_v3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6c6ad0d8",
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "matplotlib.rcParams['svg.fonttype'] = 'none'\n",
    "from collections import OrderedDict\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0b56eee4",
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAE8CAYAAADQaEpSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAeFUlEQVR4nO3de5hlVXnn8e+Pbi4GWkC6ItoNNMT2AopC2paBcRQ1k4YkjaIoF40XkEwiEUR4wowOckmYUQFHDaKYMAg+0DYq2MbmYhAiKA0U0Fyay6QhCA2ZsUU0qAFE3vlj7bJPna7L2fusU+f0Or/P85ynal/OW2uvs+utXWuvvZYiAjMz2/Rt1u8CmJlZHk7oZmaFcEI3MyuEE7qZWSGc0M3MCjG7Xz947ty5sWDBgn79eDOzTdKtt976k4gYmWhb3xL6ggULGB0d7dePNzPbJEn60WTb3ORiZlYIJ3Qzs0I4oZuZFcIJ3cysEE7oZmaFcEI3MyvEtAld0vmSfizp7km2S9LnJK2VdKekvfMX08zMptPJFfoFwJIpth8ALKxeRwPndl8sMzOra9qEHhHfB346xS4HARdGsgrYTtKLchXQzMw6k+NJ0XnAIy3L66p1/9q+o6SjSVfx7Lzzzo1/4KWXXtr4vQCHHHLI+HJ1Eat9epDX/8PxjWNd/8dnj1v+z2c2L9nVJ7SV7OrtGsdKhfnZuMUXv/gjjUM99thnxi2riw+gfX6WM993dPNgwAkXnLdh4QNd/nqc/+z45fu6ONCXjz/QT7/ykUl2nN6Jd+80bvnUbj4A4BMtH8Lf/9mCrmId+aWHxi0/MPL6xrF+b/3145bPfMlLGsc6Ye3a8SsebhwqaZ7+pjSjN0Uj4ryIWBQRi0ZGJhyKwMzMGsqR0B8FWv/kz6/WmZnZDMqR0FcAf1r1dtkH+HlEbNTcYmZmvTVtI6GkS4A3AnMlrQM+AWwOEBFfBFYCBwJrgV8B7+9VYc3MbHLTJvSIOGya7QF8KFuJzMysET8pamZWCCd0M7NCOKGbmRXCCd3MrBBO6GZmhXBCNzMrhBO6mVkhnNDNzArhhG5mVggndDOzQjihm5kVwgndzKwQTuhmZoVwQjczK4QTuplZIZzQzcwK4YRuZlYIJ3Qzs0I4oZuZFcIJ3cysEE7oZmaFcEI3MyuEE7qZWSGc0M3MCuGEbmZWCCd0M7NCOKGbmRXCCd3MrBBO6GZmhXBCNzMrhBO6mVkhnNDNzArRUUKXtETS/ZLWSjppgu07S7pW0u2S7pR0YP6impnZVKZN6JJmAecABwC7A4dJ2r1tt48DyyNiL+BQ4Au5C2pmZlPr5Ap9MbA2Ih6MiGeAZcBBbfsE8Pzq+22Bx/IV0czMOtFJQp8HPNKyvK5a1+oU4N2S1gErgb+cKJCkoyWNShpdv359g+Kamdlkct0UPQy4ICLmAwcCF0naKHZEnBcRiyJi0cjISKYfbWZm0FlCfxTYqWV5frWu1ZHAcoCIuBHYCpibo4BmZtaZThL6LcBCSbtK2oJ003NF2z4PA28GkPQKUkJ3m4qZ2QyaNqFHxLPAMcBVwL2k3ixrJJ0maWm120eBD0q6A7gEeF9ERK8KbWZmG5vdyU4RsZJ0s7N13ckt398D7Je3aGZmVoefFDUzK4QTuplZIZzQzcwK4YRuZlYIJ3Qzs0I4oZuZFcIJ3cysEE7oZmaFcEI3MyuEE7qZWSGc0M3MCuGEbmZWCCd0M7NCOKGbmRXCCd3MrBBO6GZmhXBCNzMrhBO6mVkhnNDNzArhhG5mVggndDOzQjihm5kVwgndzKwQTuhmZoVwQjczK4QTuplZIZzQzcwK4YRuZlYIJ3Qzs0I4oZuZFcIJ3cysEE7oZmaF6CihS1oi6X5JayWdNMk+75R0j6Q1ki7OW0wzM5vO7Ol2kDQLOAf4A2AdcIukFRFxT8s+C4H/CuwXEU9I+t1eFdjMzCbWyRX6YmBtRDwYEc8Ay4CD2vb5IHBORDwBEBE/zltMMzObTicJfR7wSMvyumpdq5cCL5X0A0mrJC2ZKJCkoyWNShpdv359sxKbmdmEct0UnQ0sBN4IHAZ8WdJ27TtFxHkRsSgiFo2MjGT60WZmBp0l9EeBnVqW51frWq0DVkTEryPiX4D/Q0rwZmY2QzpJ6LcACyXtKmkL4FBgRds+l5OuzpE0l9QE82C+YpqZ2XSmTegR8SxwDHAVcC+wPCLWSDpN0tJqt6uAxyXdA1wLnBgRj/eq0GZmtrFpuy0CRMRKYGXbupNbvg/g+OplZmZ94CdFzcwK4YRuZlYIJ3Qzs0I4oZuZFcIJ3cysEE7oZmaFcEI3MyuEE7qZWSGc0M3MCuGEbmZWCCd0M7NCOKGbmRXCCd3MrBBO6GZmhXBCNzMrhBO6mVkhnNDNzArhhG5mVggndDOzQjihm5kVwgndzKwQTuhmZoVwQjczK4QTuplZIZzQzcwK4YRuZlYIJ3Qzs0I4oZuZFcIJ3cysEE7oZmaFcEI3MyuEE7qZWSGc0M3MCtFRQpe0RNL9ktZKOmmK/d4uKSQtyldEMzPrxLQJXdIs4BzgAGB34DBJu0+w3xzgWOCm3IU0M7PpdXKFvhhYGxEPRsQzwDLgoAn2Ox34JPBUxvKZmVmHOkno84BHWpbXVet+S9LewE4R8Z2pAkk6WtKopNH169fXLqyZmU2u65uikjYDzgY+Ot2+EXFeRCyKiEUjIyPd/mgzM2vRSUJ/FNipZXl+tW7MHOCVwHWSHgL2AVb4xqiZ2czqJKHfAiyUtKukLYBDgRVjGyPi5xExNyIWRMQCYBWwNCJGe1JiMzOb0LQJPSKeBY4BrgLuBZZHxBpJp0la2usCmplZZ2Z3slNErARWtq07eZJ939h9sczMrC4/KWpmVggndDOzQjihm5kVwgndzKwQTuhmZoVwQjczK4QTuplZIZzQzcwK4YRuZlYIJ3Qzs0I4oZuZFcIJ3cysEE7oZmaFcEI3MyuEE7qZWSGc0M3MCuGEbmZWCCd0M7NCOKGbmRXCCd3MrBBO6GZmhXBCNzMrhBO6mVkhnNDNzArhhG5mVggndDOzQjihm5kVwgndzKwQTuhmZoVwQjczK4QTuplZIZzQzcwK0VFCl7RE0v2S1ko6aYLtx0u6R9Kdkq6RtEv+opqZ2VSmTeiSZgHnAAcAuwOHSdq9bbfbgUURsSfwdeBTuQtqZmZT6+QKfTGwNiIejIhngGXAQa07RMS1EfGranEVMD9vMc3MbDqdJPR5wCMty+uqdZM5Erhiog2SjpY0Kml0/fr1nZfSzMymlfWmqKR3A4uAT0+0PSLOi4hFEbFoZGQk5482Mxt6szvY51Fgp5bl+dW6cSS9BfgY8IaIeDpP8czMrFOdXKHfAiyUtKukLYBDgRWtO0jaC/gSsDQifpy/mGZmNp1pE3pEPAscA1wF3Assj4g1kk6TtLTa7dPANsClklZLWjFJODMz65FOmlyIiJXAyrZ1J7d8/5bM5TIzs5r8pKiZWSGc0M3MCuGEbmZWCCd0M7NCOKGbmRXCCd3MrBBO6GZmhXBCNzMrhBO6mVkhnNDNzArhhG5mVggndDOzQjihm5kVwgndzKwQTuhmZoVwQjczK4QTuplZIZzQzcwK4YRuZlYIJ3Qzs0I4oZuZFcIJ3cysEE7oZmaFcEI3MyuEE7qZWSGc0M3MCuGEbmZWCCd0M7NCOKGbmRXCCd3MrBBO6GZmhXBCNzMrhBO6mVkhOkrokpZIul/SWkknTbB9S0lfq7bfJGlB9pKamdmUpk3okmYB5wAHALsDh0navW23I4EnIuIlwGeAT+YuqJmZTa2TK/TFwNqIeDAingGWAQe17XMQ8JXq+68Db5akfMU0M7PpKCKm3kF6B7AkIo6qlt8DvC4ijmnZ5+5qn3XV8gPVPj9pi3U0cHS1+DLg/lwH0tBc4CfT7uVYvYo3DLFyxxvUWLnjDUOspvF2iYiRiTbM7r48nYuI84DzZvJnTkXSaEQscqz+xBuGWLnjDWqs3PGGIVYv4nXS5PIosFPL8vxq3YT7SJoNbAs8nqOAZmbWmU4S+i3AQkm7StoCOBRY0bbPCuC91ffvAL4X07XlmJlZVtM2uUTEs5KOAa4CZgHnR8QaSacBoxGxAvh74CJJa4GfkpL+piBn888wxModbxhi5Y43qLFyxxuGWNnjTXtT1MzMNg1+UtTMrBBO6GZmhXBCNzMrhBO6mVkhZvTBokEg6YXAvGrx0Yj4f5njbxMRv2jwvq2AvwD+IxDADcC5EfFUjRh7T7U9Im6rWaZvV2WZLN7SGrGOn2p7RJxdo2itcbcHFgJbtcT6fo3338XUx7hnzfK8YKrtEfHTGrE+z9Rl+3CNoo3F7Po8q+JkOzdaYn4FODYiflYtbw+cFREfqBsrh9znbK9+B1oNTUKX9Brgi6SHnsYejJov6WfAX9RNdlO4B9i5wfsuBJ4EPl8tHw5cBBxSI8ZZ1detgEXAHYCAPYFR4D/ULNOZ1deDgR2Br1bLhwF1/xDOqb6+DHgtG55l+BPg5pqxAJB0FHAs6WG31cA+wI3Am2qE+ePq64eqrxdVX49oUibgVlKiE+k8eKL6fjvgYWDXGrFGq6/7kQbG+1q1fAjpPGsix3kGec+NMXuOJXOAiHhC0l51Akh6kqn/0Dy/Rrjc5+yc6XfpUkQMxYv0C/+6CdbvA9xRM9bxk7w+Cvy0Yfnu6WRdh7G+CbyqZfmVwNe7qLvRTtZ1GOv7wJyW5TnA9xvGuov0x2t1tfxy4JsNY90+wbrbuqizLwMHtiwfAHypYaxVwOyW5c2BVf0+z3pwbtwBbN+y/ALgroaxTif9JzIHeD7w58BpDWNlO2d7/RqmNvStI+Km9pURsQrYumasM4DtSR9s62sbmt+XuE3SPmMLkl7Hhiu0ul4WEXeNLUTE3cArGsYC2FrSbi1l25X6dTbmhcAzLcvPVOuaeCqqpgJJW0bEfaSrqSYkab+WhX3p7h7TPhGxcmwhIq4A9m0Ya3tSUhqzTbWuiZznGeQ9N84CbpR0uqTTgR8Cn2oYa2lEfCEinoyIf4uIc9l4lNhO5TxnkfRSSddUgxoiaU9JH28ar9XQNLkAV0j6DulfzkeqdTsBfwpcWTPWbcDlEXFr+4aqGaBjLW24mwM/lPRwtbwLcF/Nco25U9LfseHf4COAOxvGAvgIcJ2kB0nNB7uwYdTMui4EbpZ0WbX8VuCChrHWSdoOuBz4rqQngB81jHUkcL6kbavlnwHdtN0+Vv2Stn4GjzWM9T+B2yVdS6r//wScUidAj84zyHhuRMSFkm4F9q9WHRwRTZuWfinpCNJw30FqCvplw1g5z1lI/72dCHwJICLulHQx8NddxASG7ElRSQcCS2m5KQqsaL2S6jDOy0hNK+sn2PbCqHGjVdIuU22PiNoJqrrx9eekX3xI/zLWvvHVFnNLUpMGwH0R8XQXsfYGXj9Wtoi4vWmslphvIN0fuTLSuP1N42wLEBE/77I8LwA+QfoMgvQZnBY1boq2xdsReF21eFNE/N+a789+nrXEznZu5FLNmvZZ0v2HAH4AHBcRDzWMl+2clXRLRLxW0u0RsVe1bnVEvKZpzN/GHqaEbvVJej4wEhEPtK3fMyK6ueofGJJeTvojf1O09FCStCQi6v73lp2kzSPi123r5kbbfAMzLde5IelVpKvWecAVwF9FxBPVtpsjYnHGYvedpCuAY4BLI2Lvas6JIyPigK6D97sRf6ZepDvUk776GOtVpJtej5AG6mm9KXRzzVjPB/4HqdfCYW3bvtCgzt5JaiZYDawBXtuyrdYNw8zHmTPWh0kTrVwOPAQc1PQYq/fMAv6MdFNu37ZtH68Za39gHWkChKuBBYNQ/z04N24AlpB6Ap1Qxfu9atvtDcr2h6QmtF3a1n+gn3XW8t7dgH8EfkVqJbihvayNY+cIsim8gPWktu8TSf8Gv6H11cdY2U5m4Buk9ta3kv64fAPYstrWJDmtBl5Ufb+Y1Nb6toZly3mcOWPdBWxTfb+AdIPw2Caxqvf8HXAxcBypC+PZLdvqJrpbgD2q798B/DPpZmtf66wH58Ydbcv7jx1rgzo7g9S89b+AB4C/7KL+s9bZBPG3pqX3TI5XtkCD/iJdOS0hzX16O+kGxB4DECvnyby6bfljpLbDHerGqt5/V9vyi6ok9eEGZct5nDljrWlb3oZ0k/zs9vrsMN6dLd/PJl3ZfRPYMkOi24P038Rb+1lnvTg3gG3b1u1Zle/xuuWi6uJZJeKVwGeq5W7rv6s6a4mzA/A50kXhraS2/h2axhsXO0eQTe1V/XK9j3SlfUw/Y2U+me8FNmtb9z7SlcWPGpTth1RXJC3r5gDXAE/38Thzxvoe8Jq2dbNJPRt+06DO7ptg3cmkP6z/XDPWKLBj27qxh6ie7Fed9eDcOJzqP4+29TsDX64Z69625Vmk+Roupe2P90zXWUuM7wL/nfSQ2a7Ax4F/bBpvXOwcQTaVV5V8D64+3FuqSp3Xz1iZT+ZPAW+ZYP2Susmket+rgYUTrN8cOKKPx5kz1vz2pNmybb8GdfZV0oTp7euPAn5dM9ZbgFdPsH474GP9qrPc50bOF/APTNDsSfov+rl+1lnL+++eYF2jB6jaX0PTy0XShaQnJlcCyyI9bNP3WNZfOcdesWYktU9pOU7UGzPoedV7/n2CbfMion0+5Bkn6WzS0AHLq1XvABZHxAldxx6ihP4cGx4saD1oARE1xnjIHCvnyZx7MKH2cTHEhnFK+nmcOWM9R+pJ8uzYqvGhYreN3zVlvIOnKds3a8SarOvfWP13PHBYzjqr4uU8N9aTepJcAtzE+M+AiPinGrGyDVDXwzoT6Yboc9WmzYBf1KmzyQzNk6IRkW2Yg5yxSANmTXoy13QmqX31CuDpLmNBag/dkXRTb1lEPNxFrJzHmTPW50g3u35QxbshurvK+TrpM1hdLY/7A0Gqy049V73nYuDbwEZXnTXkrDPIe27sCPwB6WnOw4HvAJdExJoGsUaBu0ldPWHj+q8zcFvWOouIng/ONTRX6INK0iw2nMx70sXJLOnVVZwlpLvnlwDXdJOgqicnDyZN/L0VacS/ZXWbIjIfZ7ZYVTwBb6ziLSb1+T43Iv6lQay3kurqJcC3qnKtbVKuKt7Lq3L9CWmExYuBqyPi2SnfuHGcrHVWxcxybrTF3LIq46eBUyPib2u+/zhSE8bPSY/9XxYNhrOuYmWvs5bYXQ37PKkcDfF+5XmRt/fNvqQhUu8lDVTUbdk2I109/QQ4foCOM2es7YD/UsX6YJextq7q61uk/sxvyPAZvKuq/xMHpc5ynRtk7LBQxdsN+G+kK+vltPVk6vN5dhSpe+UTwLWk/7y+1+3nEBHD0+QyyKqrkj8iXQksIDUDXNZFvBFgL9KTbuuAH3cRa9+qXK8nJaa3RcT1DWNlO85csSRtTRqF713ACKkJ4fejuyYEgKdIV4n/Rhqwaqupd5+0fPNIV8BvIyWAj9DnOmuJl+XcaOtkcGpk6GQQEQ9K+hbwPOA9wEvZ0AxWp2xZ66xyLGl89VURsX/1X9gZXcYE3OTSd5l733yA9Ej2VqS23OUR0U0yf4g06uAyUn/tcf/mR70bTAPZy0jSL0n9ipdVX8f9QkSNm5hVvDeREvBi0uPdyyKi0fC0kv6J1Ld7Oemp38fbylZn9qOsPbMynxs5OxnsRqr/g0jt38uA78QEvV46iNWT3mwtg3OtJs3R8LSkNRGxR9exndD7qwe9b+5mwxCy7cmp7l3569pjjA8XHd9gGuBeRhcw9THWGkK3KtudpCvWaI8dNaaNq5Lm2PsnOs6Oe+DkrLMq3nVkOjdyaqn/b5H+O2qv/457euWus5a4lwHvJw0P8SbSf16bR8SBTeKNi+2EXo5qCNlJRY3uX9aMpPdOtT0ivjJTZRlGkk5h6inoTp250kyvZdjnK6JtRM1G8ZzQB5eaTzi9NfDvEfFctTyLNEjXrxqWo31i4euBL0YX46u3xW90nDlJ2oE0fnnr5MmnRcTjU75xhvSsV0SXen1uDAtJD0dEk7mIxxmmKeg2RU1na7kG+J2W5eeR2nObupA0MNTngb+tvr9oynfU0/Q4c1pG6sHwdlK3t/VsmJS5NknfVZpNaWx5e0lXNYx1FGkEwauAU6uvpzQtW2a9PjcakfSVCer//D4WaTrdPhcADNGDRYNqiqc7RRr5r4mtWq94I+IXkn5nqjdM45URsXvL8rWSaiXhHh1nTi+KiNNblv9a0ru6iDcSG89g/7sNY/WsV0QGXZ8bPbLnBPW/Vx/LM50sTSVO6P13BukhiokeFGn6H9QvJe091tNA0u/T3VOGt0naJ9KE2qjZxMK9OM6crpZ0KOPH12h0RV35jaSdx7o/Kk0B1/SX9qmIeEoSqibDVpoGcRDkODd6YTNJ28eGmY9eQJ/z3Uxc1Dih91+2CadbHAdcKukx0smyI6mfdS3KO7FwL46za23jaxzHhuaCWcAvSBMbNPEx4Iaq66FIfbWbTqydczLsLDKfG71wFnCjpEur5UOAv+ljeSB1QZ3MZ3P8AN8U7TNlnHC67b2bA2NXcfc3uYOujBML9+o4B5mkuaTJECA1l3Q9B6gyTYadoRzZzo1ekbQHaZweSE9iDkJTUE85oRdGQzCpcy8o80TMueNZOSTNJ91E3q9adT1p2sN13cYehLbLoSZpxVSvmrHeSfp39xuS1kh6bcvmCxqU7VWSVkl6RNJ5Vde5sW0314yV7ThzkrS/pHXAv0q6WtKuLZuvzhBvQdN4Oes/t0Et26CWq83/Js35++Lq9e1qXfdyDAjjV1cD9eSccHo1mSburd6TczLmbMeZuf6zTcScO17O+u9BvQ1k2Qa1XG1lXN3Jukax+31ww/4i74TT2Sburd6fczLmbMeZuf6zTcScO17O+p+BehuIsg1qudrKdA3w7up3Ylb1/TVZYvf74Pwa90F3O+F0tol7q/f2apLcrMO3dlnn2SZizh2vV/Wfqd4GsmyDWq628uxCanJZTxoJ9XJg5yyx+31wfv02weWYcPrVZJy4l/wTC2cd8zpT3WebiDl3vNz1n7neBrJsg1qumXq5l0ufaUgmnB6W4zSbjKSTp9gcMf5J5WY/wwm9v5R3KNhsE/dW8XJPxpx9KNJuKeNEzLnj5az/3Aa1bINaLgBJH51g9dbAkcAOEdH106J+UrTPIu+E0zkn7oWMk+RmPs6cck7EnDte7omdcxrUsg1quYiIs8a+lzSHNEbP+0kDw5012fvq8BV6YZRx4l71cJLcQaJMEzHnjjfI9T+oZRvUco2pxpQ5HjiC1OPrs1GNN5NFvxvx/erNi4yTOlfxBqZnSo/rLctEzLnjDXL9D2rZBq1cpMHpHgD+CtimFz/DV+iF0cYT934tGk7qXMVrnyR3BXB+RDzafWkHgzaeiHk5cFk0nHQjZ7xBrv9BLdsAl+s54GnSiKM9uY/khF4QZZy4t4pXfM8UZZyIOXe8Qa7/QS3boJZrpjihF0SZJ+4d1J4pOSnjRMy54w1y/Q9q2Qa1XDPFCd3MrBDutlggzcDEvRqAiZ1zU+aJmHPHM5uOr9ALJGk58CTw1WrV4cB2EXFIxp+RZZbyQVHNmnQsG8Zd2Qe4sW4zVa/imXXCV+hlyjJxrwZ/YuecjiXvRMy545lNa1Cf3rPu3CZpbOqzbibuPQPYntRro/W1DeWdO0+NNUmpmoiZDVP4DUI8s2n5Cr0gyj9x70BO7NwjuSdiHriJna18bkMviDJP3KshnNgZ8k/EnDue2WSc0M3MClFaO+hQyz1B7qBO7JxTD+psU5ik2ArlNvSynAucAqwCjgJukLQ0Ih4gtavXNbBDkWaUu85yxzPrmBN6WeZExJXV92dKuhW4UtJ7mHxIgKnsyIahSA9nwIYizSR3neWOZ9YxN7kUphoPHYCIuBZ4O3ARqadLLRHxm4i4MiLeS3owZi1wnaRjcpV3EOSss17EM+uUE3pZPgm8onVFRNwJvJk0i1FtkraUdDDpqdMPAZ8DLuuynIMkd51l/wzMOuVeLjapYR+K1GxT44RekNwT5A7DUKQ9qLOBnaTYyuebomXJ2islBndi55xy9+QZhp5BNqB8hV6QQZ8gdxDlrjN/BtZPw3AFNjSGpVdKTrnrzJ+B9ZObXAozwQS5pfVKyS53nfkzsH5xk0tB3Culvtx15s/A+skJvSDD0Cslt9x15s/A+skJ3cysEL4pOiQklTZlnJm1cUIfHrXnFDWzTYt7uRRkyCZ1NrM2vkIvyzBN6mxmbXyFXpZhmtTZzNq4l0tBhnVSZzNLnNDNzArhJpeCeOhWs+HmhF4WD91qNsTc5FIQD91qNtzcla0gHrrVbLi5yaUwHrrVbHi5yaUgHrrVbLg5oRfEQ7eaDTcndDOzQvimqJlZIZzQzcwK4YRuZlYIJ3Qzs0L8f+M7Q9JN215YAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "colors = {\n",
    "\"M0\"\t:\t(170./255,170./255,170./255),\n",
    "\"M1\"\t:\t(0/255,255./255,255./255),\n",
    "\"M2a\"\t:\t(0/255,0/255,255./255),\n",
    "\"M2b\"\t:\t(138./255,43./255,226./255),\n",
    "\"M2c\"\t:\t(156./255,102./255,31./255),\n",
    "\"M2d\"\t:\t(139./255,35./255,35./255),\n",
    "\"M2aM2cM2d\"\t:\t(255./255,97./255,3./255),\n",
    "\"M1M2bM2cM2d\"\t:\t(69./255,139./255,0./255),\n",
    "\"M2cM2d\"\t:\t(220./255,20./255,60./255),\n",
    "\"M1M2bM2d\"\t:\t(255./255,185./255,15./255),\n",
    "\"M2aM2d\"\t:\t(255./255,215./255,0./255),\n",
    "\"M2aM2bM2d\"\t:\t(139./255,95./255,101./255),\n",
    "\"M2bM2d\"\t:\t(128./255,0./255,0./255),\n",
    "\"M1M2b\"\t:\t(60./255,179./255,113./255),\n",
    "\"M1M2d\"\t:\t(25./255,25./255,112./255),\n",
    "\"NoLabel\"\t:\t(255./255,225./255,255./255)\n",
    "\n",
    "            }\n",
    "\n",
    "colors = OrderedDict(sorted(colors.items(), key=lambda t: t[0]))\n",
    "\n",
    "\n",
    "plt.bar(range(len(colors)), [1]*len(colors), color=colors.values() )\n",
    "plt.xticks(np.arange(0.5,len(colors)+.5,1), list(colors.keys()), rotation='vertical')\n",
    "plt.savefig('ColorCode.svg')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1f885561",
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "def labelState(x):\n",
    "    #x = [STAT3,SMAD23,STAT6,SOCS1,TLR4,NFKB,STAT1,SOCS3,AP1,IL10,TNFA,  TGFB,  Fra1, HIF1A, ERK,  IL4e, IL10e, TNFAe, TGFBe,  IgG,   A2a,   IFNG,  IFNB,  IL1B,  IL6e,  MCSF,   GMCSF,  Hipoxia,  GCGCR] \n",
    "    #x    x[0]   x[1]   x[2]  x[3] x[4]  x[5] x[6]  x[7] x[8] x[9] x[10] x[11]  x[12] x[13]  x[14] x[15] x[16]  x[17]   x[18]  x[19]  x[20]  x[21]  x[22]  x[23]  x[24]  x[25]   x[26]   x[27]     x[28]\n",
    "    label = ''\n",
    "    feno = ''\n",
    "    fl = 0\n",
    "    if (x[0]<=0.25) and (x[2]<=0.25) and (x[5]<=0.25) and (x[6]<=0.25) and (x[8]<=0.25) and (x[12]<=0.25) and (x[13]<=0.25) and (x[14]<=0.25): \n",
    "        label = (\"M0\")\n",
    "        fl += 1\n",
    "        feno += label\n",
    "        feno += '-'\n",
    "########### \n",
    "    if (x[2]>=.75) and (((x[23]>=.75) and (x[8]>=0.75)) or (x[14]>=0.75)) and (((x[4]>=0.75) and (x[20]>=0.75)) or ((x[12]>=0.75) and (x[8]>=0.75)) or (x[13]>=0.75) or (x[12]>=0.75)):\n",
    "            label = (\"M2aM2bM2d\")\n",
    "            fl += 1\n",
    "            feno += label\n",
    "            feno += '-'\n",
    "    else:\n",
    "        if (x[2]>=.75) and (x[0]>=.75) and (((x[4]>=0.75) and (x[20]>=0.75)) or ((x[12]>=0.75) and (x[8]>=0.75)) or (x[13]>=0.75) or (x[12]>=0.75)):\n",
    "            label = (\"M2aM2cM2d\")\n",
    "            fl += 1\n",
    "            feno += label\n",
    "            feno += '-'\n",
    "        elif (x[0]>=.75) and (((x[4]>=0.75) and (x[20]>=0.75)) or ((x[12]>=0.75) and (x[8]>=0.75)) or (x[13]>=0.75) or (x[12]>=0.75)):\n",
    "            label = (\"M2cM2d\")\n",
    "            fl += 1\n",
    "            feno += label\n",
    "            feno += '-'\n",
    "        elif (x[2]>=.75) and (((x[4]>=0.75) and (x[20]>=0.75)) or ((x[12]>=0.75) and (x[8]>=0.75)) or (x[13]>=0.75) or (x[12]>=0.75)):\n",
    "            label = (\"M2aM2d\")\n",
    "            fl += 1\n",
    "            feno += label\n",
    "            feno += '-'\n",
    "        elif (x[2]>=.75):\n",
    "            label = (\"M2a\")\n",
    "            fl += 1\n",
    "            feno += label\n",
    "            feno += '-'\n",
    "        elif (x[0]>=.75):\n",
    "            label = (\"M2c\")\n",
    "            fl += 1\n",
    "            feno += label\n",
    "            feno += '-'\n",
    "        elif ((x[4]>=0.75) and (x[20]>=0.75)) or ((x[12]>=0.75) and (x[8]>=0.75)) or (x[13]>=0.75) or (x[12]>=0.75):\n",
    "            label = (\"M2d\")\n",
    "            fl += 1\n",
    "            feno += label\n",
    "            feno += '-'\n",
    "        else:\n",
    "            if ((x[5]>=0.75) or (x[6]>=0.75) or ((x[10]>=0.75) and (x[8]>=0.75)) or ((x[17]>=0.75) and (x[8]>=0.75))) and (((x[23]>=.75) and (x[8]>=0.75)) or (x[14]>=0.75)) and (x[0]>=.75) and (((x[4]>=0.75) and (x[20]>=0.75)) or ((x[12]>=0.75) and (x[8]>=0.75)) or (x[13]>=0.75) or (x[12]>=0.75)):\n",
    "                label = (\"M1M2bM2cM2d\")\n",
    "                fl += 1\n",
    "                feno += label\n",
    "                feno += '-'\n",
    "            elif ((x[5]>=0.75) or (x[6]>=0.75) or ((x[10]>=0.75) and (x[8]>=0.75)) or ((x[17]>=0.75) and (x[8]>=0.75))) and (((x[23]>=.75) and (x[8]>=0.75)) or (x[14]>=0.75)) and (((x[4]>=0.75) and (x[20]>=0.75)) or ((x[12]>=0.75) and (x[8]>=0.75)) or (x[13]>=0.75) or (x[12]>=0.75)):\n",
    "                label = (\"M1M2bM2d\")\n",
    "                fl += 1\n",
    "                feno += label\n",
    "                feno += '-'\n",
    "            elif ((x[5]>=0.75) or (x[6]>=0.75) or ((x[10]>=0.75) and (x[8]>=0.75)) or ((x[17]>=0.75) and (x[8]>=0.75))) and (((x[23]>=.75) and (x[8]>=0.75)) or (x[14]>=0.75)):\n",
    "                label = (\"M1M2b\")\n",
    "                fl += 1\n",
    "                feno += label\n",
    "                feno += '-'\n",
    "            elif ((x[5]>=0.75) or (x[6]>=0.75) or ((x[10]>=0.75) and (x[8]>=0.75)) or ((x[17]>=0.75) and (x[8]>=0.75))) and (((x[4]>=0.75) and (x[20]>=0.75)) or ((x[12]>=0.75) and (x[8]>=0.75)) or (x[13]>=0.75) or (x[12]>=0.75)):\n",
    "                label = (\"M1M2d\")\n",
    "                fl += 1\n",
    "                feno += label\n",
    "                feno += '-'\n",
    "            elif (((x[23]>=.75) and (x[8]>=0.75)) or (x[14]>=0.75)) and (((x[4]>=0.75) and (x[20]>=0.75)) or ((x[12]>=0.75) and (x[8]>=0.75)) or (x[13]>=0.75) or (x[12]>=0.75)):\n",
    "                label = (\"M2bM2d\")\n",
    "                fl += 1\n",
    "                feno += label\n",
    "                feno += '-'\n",
    "            elif ((x[23]>=.75) and (x[8]>=0.75)) or (x[14]>=0.75): \n",
    "                label = (\"M2b\")\n",
    "                fl += 1\n",
    "                feno += label\n",
    "                feno += '-'\n",
    "            elif (x[5]>=0.75) or (x[6]>=0.75) or ((x[10]>=0.75) and (x[8]>=0.75)) or ((x[17]>=0.75) and (x[8]>=0.75)): \n",
    "                label = (\"M1\")\n",
    "                fl += 1\n",
    "                feno += label\n",
    "                feno += '-'\n",
    "############\n",
    "    if (label == \"\"): label = \"NoLabel\"\n",
    "#    print(feno,fl)\n",
    "    return label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "40e188ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_simulation_csv(df):\n",
    "    df = pd.read_csv(name, index_col=0)\n",
    "    df[\"Label\"] = df.apply(labelState, axis=1)\n",
    "    name = name.replace(\"-A\",\"A\").replace(\"-B\",\"B\").replace(\"-C\",\"C\").replace(\"-\",\"_\").replace(\".csv\",\"\")\n",
    "    path = name.split(\"/\")[0] + \"/\"\n",
    "    name = name.split(\"/\")[1].split(\"_\")\n",
    "    name = \"_\".join([name[0], df[\"Label\"].iloc[0], df[\"Label\"].iloc[-1], name[-1]]) + \".csv\"\n",
    "    df.to_csv(path+'Clean/'+name)\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "db388775",
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "def analyse_simulation_csv(path,name, threshold=.25, sep=','):\n",
    "    \"\"\"\n",
    "    Toma un csv con los resultados de la simulacion\n",
    "    Regresa:\n",
    "        * Entorno\n",
    "        * Etiqueta inicial\n",
    "        * Etiqueta final\n",
    "        * Nodo perturbado\n",
    "        * Transicion\n",
    "    En el nombre del archivo \n",
    "        * primero es el entorno, \n",
    "        * segundo el estado inicial, \n",
    "        * ultimo el gen perturbado\n",
    "    En el archivo:\n",
    "        * Primera fila genes\n",
    "        * Primera columna valor perturbado\n",
    "        * Penultima fila differencia con vector inicial\n",
    "        * Ultima fila etiqueta\n",
    "    Si el nodo modificado es un input (termina con e) al calcular la diferencia se ignora el gen modificado, \n",
    "        para compenzar el error de que su expresión es constante.\n",
    "        \"\"\"\n",
    "    df = pd.read_csv(path+\"/csv/\"+name, index_col=0)\n",
    "    env  = name.replace('.csv','').split('_')[0]\n",
    "    node = name.replace('.csv','').split('_')[-1]\n",
    "    # Clean and rewrite csv\n",
    "    if node[-1]=='e': #if node is an input ignore value of input, as it is constant\n",
    "        first_row = df.drop(node.split('+'), axis=1).iloc[[0]].values[0]\n",
    "        df[\"Sum\"] = df.drop(node.split('+'), axis=1).apply(lambda row: sum(abs(row - first_row)), axis=1)\n",
    "    else:\n",
    "        first_row = df.iloc[[0]].values[0]\n",
    "        df[\"Sum\"] = df.apply(lambda row: sum(abs(row - first_row)), axis=1)\n",
    "    df[\"Label\"] = df.apply(labelState, axis=1)\n",
    "    df.to_csv(path+\"/csv-clean/\"+name)\n",
    "    # Create output for TransitionTable file\n",
    "    init = df[\"Label\"].iloc[0]\n",
    "    final = df[\"Label\"].iloc[-1]\n",
    "    labels = set(df[\"Label\"].unique())\n",
    "    if len(labels) == 1: \n",
    "        transition = 'None'\n",
    "        labels = 1\n",
    "    else:\n",
    "        if \"NoLabel\" in labels: labels = len(labels) - 1\n",
    "        else: labels = len(labels)\n",
    "        first_row = df.drop('Label', axis=1).iloc[[0]].values[0]\n",
    "        df = df.set_index(np.arange(0,len(df),1))\n",
    "        df[\"Sum\"] = df.drop('Label', axis=1).apply(lambda row: sum(abs(row - first_row)), axis=1)\n",
    "        df[\"Diff\"] = df[\"Sum\"].diff().abs()\n",
    "        jumps = df.index[(df[\"Diff\"] >= threshold)]\n",
    "        if len(jumps) == 0: transition = 'Soft'\n",
    "        else: transition = 'Hard_' + str(len(jumps))    \n",
    "    return [env, init, final, node, transition, str(labels)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9742a0ab",
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def plot_simulation_csv(path,name, colors, unique=True):\n",
    "    \n",
    "    df = pd.read_csv(path+\"/csv-clean/\"+name, index_col=0)\n",
    "    #print( name )\n",
    "    if unique and len(df[\"Label\"].unique()) > 1: return\n",
    "    #print(df[\"Label\"].unique())\n",
    "#    print( df[\"Label\"].map(colors))\n",
    "    df[\"Color\"] = df[\"Label\"].map(colors)\n",
    "    #df = df.iloc[::4, :] #select only some points\n",
    "    title = name.replace('.csv','')\n",
    "    fig = plt.figure()\n",
    "    plt.figure(figsize=(3,2))\n",
    "    #plt.xlabel(title.split('_')[-1])\n",
    "    plt.gcf().text(0.5, -0.1, title.split('_')[-1], fontsize=10,rotation=0)\n",
    "    plt.ylabel(\"Difference\")\n",
    "    plt.axis([-.1, 1.1, df[\"Sum\"].min()-.3, df[\"Sum\"].max()+.3])\n",
    "    plt.text(-0.05,df[\"Sum\"].min()+0,df[\"Label\"].iloc[0])\n",
    "    plt.text(1.1-len(df[\"Label\"].iloc[-1])*0.005,df[\"Sum\"].max()*.8,df[\"Label\"].iloc[-1],ha=\"right\")\n",
    "    plt.scatter(df.index, df.Sum, c=df.Color)\n",
    "    plt.savefig(path+\"/svg-color/\"+name.replace('.csv','.svg'))\n",
    "    plt.close()\n",
    "    #return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ecf1d7bf",
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Lenovo\\Documents\\Python\n",
      "C:\\Users\\Lenovo\\Documents\\Python/Results-M2bM2d-env\n"
     ]
    }
   ],
   "source": [
    "path = os.getcwd()\n",
    "print(path)\n",
    "folder = \"Results-M2bM2d-env\"\n",
    "path = path+\"/\"+folder\n",
    "print(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "da43f136",
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "78"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#path = \".\"\n",
    "f = open(path +'/TransitionTable.csv','w')\n",
    "f.write(\"Environment,Init label, Final label, Perturbed node, Transition,Number labels\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "241ecad6",
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['M2bM2dProb_A2a+Hipoxia+GCGCR.csv',\n",
       " 'M2bM2dProb_IFNG+IFNB.csv',\n",
       " 'M2bM2dProb_IgG+GCGCR.csv',\n",
       " 'M2bM2dProb_IL10e+IL6e+MCSF.csv',\n",
       " 'M2bM2dProb_IL1B+TNFAe.csv',\n",
       " 'M2bM2dProb_IL4e+TGFBe.csv']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(path+\"/csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0bfd7d23",
   "metadata": {
    "jupyter": {
     "outputs_hidden": false
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "M2bM2dProb_A2a+Hipoxia+GCGCR.csv\n",
      "M2bM2dProb_IFNG+IFNB.csv\n",
      "M2bM2dProb_IgG+GCGCR.csv\n",
      "M2bM2dProb_IL10e+IL6e+MCSF.csv\n",
      "M2bM2dProb_IL1B+TNFAe.csv\n",
      "M2bM2dProb_IL4e+TGFBe.csv\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for file in os.listdir(path+\"/csv\"):\n",
    "    print(file)\n",
    "    if not os.path.exists(path+\"/csv-clean\"): os.makedirs(path+\"/csv-clean\")\n",
    "    f.write( ','.join( analyse_simulation_csv(path,file) )+'\\n' )\n",
    "    if not os.path.exists(path+\"/svg-color\"): os.makedirs(path+\"/svg-color\")\n",
    "    plot_simulation_csv(path,file, colors, unique=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1c9e24e",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3dfe152c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}