predictions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "c06578c5",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "de922321",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sys import version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7fd92fd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(action='once')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "238022ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "\n",
    "from pathlib import Path\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import LinearSVC #???????????????//\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "from sklearn.metrics import mean_squared_error, r2_score, confusion_matrix\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from scipy.stats import pearsonr, kendalltau\n",
    "\n",
    "import astropy.table\n",
    "from astropy.table import QTable, join\n",
    "from time import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "16bd66a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "master_cat = pd.read_csv('./catdata/master_catalog_jan_2023.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1d22fd27",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_cat(field): # change to match-case\n",
    "    cat_files = ['cat1_50.pk','cat51_100.pk','cat101_150.pk','cat151_200.pk','cat201_235.pk',\n",
    "             'cat236_257.pk','cat258_279.pk','cat280_320.pk','cat321_360.pk','cat361_406.pk']\n",
    "    bounds = [50,100,150,200,235,257,279,320,360,406]\n",
    "    for b in range(len(bounds)):\n",
    "        if field <= bounds[b]:\n",
    "            to_load = cat_files[b]\n",
    "            break\n",
    "    with open(f'./pickle/{to_load}','rb') as f:\n",
    "        catalogue = pickle.load(f)\n",
    "    return catalogue[field]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3340473d",
   "metadata": {},
   "source": [
    "### Load Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "35df8ca6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# select most recent training data\n",
    "train_file = 'training_data_0802.pk' # training data with 3 classes\n",
    "train_file = 'training_data_1702.pk' # training data with only gcs and galaxies\n",
    "#train_file = 'training_data_1902_with_stars.pk' # training data with gcs galaxies and stars, classed as 'gc' and 'non-gc'\n",
    "#train_file = 'training_data_2702_j.pk'\n",
    "train_file = 'training_data_0203_jhk.pk'\n",
    "train_file = 'training_data_2704.pk'\n",
    "\n",
    "with open(f'./pickle/training_data/{train_file}','rb') as f:\n",
    "    training_data = pickle.load(f)\n",
    "\n",
    "#training_data = training_data[training_data['j_acc']==True]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7dd4eb30",
   "metadata": {},
   "source": [
    "### Add i-g to training data (17/02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "dfdacdb1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#training_data['i-g'] = training_data['i']-training_data['g']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d35d779",
   "metadata": {},
   "source": [
    "# Training Data Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0890fddc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 23.01.26 18:29\n",
    "def generate_training_data(matches:dict, crowding=300) -> pd.DataFrame:\n",
    "    cat = load_cat(1)\n",
    "    \n",
    "    columns = ['obj_id','class','i','g','di','dg','ra','dec','field','pdidx','rbcidx','nearby']\n",
    "    values = []\n",
    "    object_ids = []\n",
    "    \n",
    "    #TEMP\n",
    "    crowded_objects = []\n",
    "    \n",
    "    \n",
    "    for field in matches: # iterate through each field ID\n",
    "        working_field = matches[field] # take the list of matches e.g. working_field = [(166727, 2642), (159637, 2646)]\n",
    "        if field not in cat: # load the correct catalogue\n",
    "            cat = load_cat(field)\n",
    "        for m in working_field: # iterate through each match (a tuple) and grab values from catalogues\n",
    "            \n",
    "            if m[2] > crowding: # testing\n",
    "                crowded_objects.append(m[1:])\n",
    "                continue\n",
    "            \n",
    "            obj_id = master_cat.loc[m[1]].ID\n",
    "            class_ = master_cat.loc[m[1]].CLASS\n",
    "            \n",
    "            if obj_id in object_ids: continue # if we've already added the object then skip\n",
    "            else: object_ids.append(obj_id)   # else add it to the list of ids\n",
    "            \n",
    "            if class_ == 1: class_str = 'gc' # convert class numbers into strings\n",
    "            elif class_ == 8: class_str = 'gc' # include extended clusters\n",
    "            elif class_ == 4: class_str = 'galaxy'\n",
    "           # elif class_ == 6: class_str = 'star'\n",
    "            else: continue # skip non-gc/gal objects\n",
    "            \n",
    "            # collect required data\n",
    "            row = cat[field][m[0]]\n",
    "            ra = row['RA']\n",
    "            dec = row['Dec']\n",
    "            g = row['g']\n",
    "            i = row['i']\n",
    "            dg = row['dg']\n",
    "            di = row['di']\n",
    "            \n",
    "            values.append([obj_id,class_str,i,g,di,dg,ra,dec,field,m[0],m[1],m[2]])\n",
    "    \n",
    "    training_data_dict = dict(zip(columns,zip(*values))) # zip values and columns together into a dict (columns as keys)\n",
    "    training_data_df = pd.DataFrame(training_data_dict) # put into pd Dataframe\n",
    "    return training_data_df, crowded_objects"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e39fbceb",
   "metadata": {},
   "source": [
    "#### Generate training data from object matches (17/02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "3fada3ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'./pickle/matches/matches_delta005_1702.pk','rb') as f:\n",
    "    obj_mat = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a1831b9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading cat1_50.pk ...\n",
      "Loading cat51_100.pk ...\n",
      "Loading cat101_150.pk ...\n",
      "Loading cat151_200.pk ...\n",
      "Loading cat201_235.pk ...\n",
      "Loading cat236_257.pk ...\n",
      "Loading cat258_279.pk ...\n",
      "Loading cat280_320.pk ...\n",
      "Loading cat321_360.pk ...\n",
      "Loading cat361_406.pk ...\n"
     ]
    }
   ],
   "source": [
    "new_training_data, crowded_obj = generate_training_data(obj_mat,crowding=350)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "68ad9a32",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f'./pickle/training_data/training_data_1702.pk','wb') as f:\n",
    "    pickle.dump(new_training_data,f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5198aec3",
   "metadata": {},
   "source": [
    "# Statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3bfbf62c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_correlations(pred,true):\n",
    "    correlations = {}\n",
    "    correlations['mse'] = mean_squared_error(pred,true)\n",
    "    correlations['ktau'] = kendalltau(pred,true)[0]\n",
    "    correlations['pval-ktau'] = kendalltau(pred,true)[1]\n",
    "    correlations['pearsonr'] = pearsonr(pred,true)[0]\n",
    "    correlations['pval-pearsonr'] = pearsonr(pred,true)[1]\n",
    "    correlations['r2'] = r2_score(true, pred)\n",
    "    return correlations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a7d550ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pretty_corr(c): # pretty print output from calc_correlations()\n",
    "    print(f\"\"\"\n",
    "    Mean squared error (RMS): \\t{c['mse']:.5f}\\t({(c['mse']**.5):.5})\n",
    "    Kendall Tau: \\t\\t{c['ktau']:.5}\n",
    "    \\tKtau p-value: \\t\\t{c['pval-ktau']:.5}\n",
    "    Pearson's r: \\t\\t{c['pearsonr']:.5}\n",
    "    \\tPearson's r p-value: \\t{c['pval-pearsonr']:.5}\n",
    "    Coef. of determination \\t{c['r2']:.5}\n",
    "    \"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "012ae5bd",
   "metadata": {},
   "source": [
    "# Machine Learning Models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b82089e",
   "metadata": {},
   "source": [
    "## Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1237338f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate the classifier and return (with optional returning of train and test values)\n",
    "def ranfor(df,train_size=0.8,n_estimators=50,criterion='gini',features=['i','g','g-i'], max_depth=None, \n",
    "           max_leaf_nodes=None, min_samples_leaf=1, stats=False):\n",
    "    # select features for training\n",
    "    X = df[features]\n",
    "    y = df['class']\n",
    "    # split the data\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=train_size) # X\n",
    "    # train the regressor model\n",
    "    ran_for_class = RandomForestClassifier(n_estimators=n_estimators, max_depth=max_depth,\n",
    "                                    criterion=criterion, max_leaf_nodes=max_leaf_nodes,\n",
    "                                    min_samples_leaf=min_samples_leaf,n_jobs=-1                                        \n",
    "                                   ).fit(X_train,y_train)\n",
    "    \n",
    "    \n",
    "    #acc = ran_for_class.score(X_test,y_test)\n",
    "    \n",
    "    #true = y_test.to_numpy()\n",
    "    if stats:\n",
    "        train_pred = ran_for_class.predict(X_train)\n",
    "        test_pred = ran_for_class.predict(X_test)\n",
    "        return ran_for_class, test_pred, y_test, train_pred, y_train\n",
    "    else: return ran_for_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "37721bce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# returns predictions for a given field, allowing a crowding parameter to filter training values\n",
    "def rf_pred(field:int,train:pd.DataFrame,crowding=300,n_estimators=50,max_depth=None,max_leaf_nodes=None,min_samples_leaf=1,features=['i','g','g-i'],scale=False):\n",
    "    training_data_ = train[train['nearby'] <= crowding]\n",
    "    cat = load_cat(field)\n",
    "    \n",
    "    # drop rows with high delta g/i values\n",
    "    cat_d = cat[cat['dg']+cat['di'] < 0.05]\n",
    "    # drop stars & saturated points\n",
    "    cat_candidate = cat_d[(cat_d['ig'] == 1) & (cat_d['ii'] == 1)]\n",
    "    # add in g-i feature\n",
    "    cat_candidate['g-i'] = cat_candidate['g']-cat_candidate['i']\n",
    "    \n",
    "    X = cat_candidate[features]\n",
    "    X = X.to_pandas()\n",
    "    if scale:\n",
    "        X_scaled = preprocessing.StandardScaler().fit(X).transform(X)\n",
    "        res = ranfor(training_data_,train_size=0.8,n_estimators=n_estimators,criterion='gini',\n",
    "                     features=features, max_depth=max_depth, max_leaf_nodes=max_leaf_nodes,\n",
    "                     min_samples_leaf=1, scale=True).predict(X_scaled)\n",
    "    else:\n",
    "        res = ranfor(training_data_,train_size=0.8,n_estimators=n_estimators,criterion='gini',\n",
    "                     features=features, max_depth=max_depth, max_leaf_nodes=max_leaf_nodes,\n",
    "                     min_samples_leaf=1).predict(X)\n",
    "    \n",
    "    cat_pred = cat_candidate[['RA','Dec','iccd','xg','yg','g','dg','ig','xi','yi','i','di','ii','field']]\n",
    "    cat_pred['pred'] = res\n",
    "    return cat_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78804065",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8cba3a43",
   "metadata": {},
   "outputs": [],
   "source": [
    "# make a plot of the different statistics from a dictionary\n",
    "def plot_stats(stats: dict, xlabel: str):\n",
    "    keys_ = stats.keys()\n",
    "    acc = [stats[k]['acc'] for k in keys_]\n",
    "    prec = [stats[k]['prec'] for k in keys_]\n",
    "    rec = [stats[k]['rec'] for k in keys_]\n",
    "    plt.plot(keys_, acc, label='accuracy')\n",
    "    plt.plot(keys_, prec, label='precision')\n",
    "    plt.plot(keys_, rec, label='recall')\n",
    "    plt.xlabel(xlabel)\n",
    "    plt.ylabel('score')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b4dd55c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "5fc96157",
   "metadata": {},
   "source": [
    "# Make Predictions with J, H, K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dcb6aee8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# select most recent training data\n",
    "train_file = 'training_data_0802.pk' # training data with 3 classes\n",
    "train_file = 'training_data_1702.pk' # training data with only gcs and galaxies\n",
    "#train_file = 'training_data_1902_with_stars.pk' # training data with gcs galaxies and stars, classed as 'gc' and 'non-gc'\n",
    "train_file = 'training_data_2702_j.pk'\n",
    "train_file = 'training_data_0203_jhk.pk'\n",
    "\n",
    "with open(f'./pickle/training_data/{train_file}','rb') as f:\n",
    "    training_data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "38d203dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter out inaccurate 2mass values\n",
    "training_data = training_data[training_data['2mass_acc']==True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "08c53e5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Field 211\n",
      "Filtering...\n",
      "2024\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "field = 211\n",
    "date = '1303'\n",
    "n_trees = 40\n",
    "max_depth_ = 9\n",
    "max_leaf_nodes_ = 12\n",
    "min_samples_leaf_ = 15\n",
    "features_ = ['i','g','i-g','j','h','k']\n",
    "features_ = ['i','g','i-g']\n",
    "\n",
    "Path(f'./pickle/predictions/{date}').mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "print(f'Field {field}')\n",
    "predictions = rf_pred(field,training_data,crowding=250,n_estimators=n_trees,max_depth=max_depth_, max_leaf_nodes=max_leaf_nodes_,\n",
    "                      min_samples_leaf=min_samples_leaf_, features=features_, filtered=False)\n",
    "print('Filtering...')\n",
    "\n",
    "gc_candidates = predictions[predictions['pred']=='gc']\n",
    "with open(f'pickle/predictions/{date}/predictionsf{field}.pk','wb') as f:\n",
    "    pickle.dump(gc_candidates,f)\n",
    "print(len(gc_candidates))\n",
    "print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "4a0458e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><i>Table length=5</i>\n",
       "<table id=\"table2156964321072\" class=\"table-striped table-bordered table-condensed\">\n",
       "<thead><tr><th>RA</th><th>Dec</th><th>iccd</th><th>xg</th><th>yg</th><th>g</th><th>dg</th><th>ig</th><th>xi</th><th>yi</th><th>i</th><th>di</th><th>ii</th><th>field</th><th>pred</th></tr></thead>\n",
       "<thead><tr><th>float32</th><th>float32</th><th>uint8</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>int8</th><th>float32</th><th>float32</th><th>float32</th><th>float32</th><th>int8</th><th>uint16</th><th>object</th></tr></thead>\n",
       "<tr><td>15.404491</td><td>42.619797</td><td>1</td><td>861.24</td><td>813.23</td><td>19.06</td><td>0.002</td><td>1</td><td>863.88</td><td>813.38</td><td>18.291</td><td>0.001</td><td>1</td><td>274</td><td>gc</td></tr>\n",
       "<tr><td>14.476671</td><td>41.698032</td><td>34</td><td>1902.57</td><td>726.48</td><td>18.773</td><td>0.001</td><td>1</td><td>1901.58</td><td>727.47</td><td>17.9</td><td>0.001</td><td>1</td><td>274</td><td>gc</td></tr>\n",
       "<tr><td>14.4192</td><td>42.163963</td><td>17</td><td>1591.71</td><td>4634.85</td><td>22.548</td><td>0.012</td><td>1</td><td>1595.62</td><td>4635.87</td><td>22.879</td><td>0.036</td><td>1</td><td>274</td><td>gc</td></tr>\n",
       "<tr><td>15.366962</td><td>42.15539</td><td>19</td><td>1797.06</td><td>4560.16</td><td>18.648</td><td>0.001</td><td>1</td><td>1796.23</td><td>4559.79</td><td>17.654</td><td>0.001</td><td>1</td><td>274</td><td>gc</td></tr>\n",
       "<tr><td>15.050808</td><td>42.14054</td><td>21</td><td>2098.96</td><td>4256.69</td><td>22.807</td><td>0.014</td><td>1</td><td>2098.46</td><td>4256.77</td><td>22.836</td><td>0.036</td><td>1</td><td>274</td><td>gc</td></tr>\n",
       "</table></div>"
      ],
      "text/plain": [
       "<Table length=5>\n",
       "    RA       Dec     iccd    xg      yg   ...    i       di    ii  field   pred \n",
       " float32   float32  uint8 float32 float32 ... float32 float32 int8 uint16 object\n",
       "--------- --------- ----- ------- ------- ... ------- ------- ---- ------ ------\n",
       "15.404491 42.619797     1  861.24  813.23 ...  18.291   0.001    1    274     gc\n",
       "14.476671 41.698032    34 1902.57  726.48 ...    17.9   0.001    1    274     gc\n",
       "  14.4192 42.163963    17 1591.71 4634.85 ...  22.879   0.036    1    274     gc\n",
       "15.366962  42.15539    19 1797.06 4560.16 ...  17.654   0.001    1    274     gc\n",
       "15.050808  42.14054    21 2098.96 4256.69 ...  22.836   0.036    1    274     gc"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc_candidates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0e86ef7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Field 274\n",
      "Filtering...\n",
      "1618\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "field = 274\n",
    "n_trees = 40\n",
    "max_depth_ = 9\n",
    "max_leaf_nodes_ = 12\n",
    "min_samples_leaf_ = 20\n",
    "features_ = ['i','g','i-g','j','h','k']\n",
    "features_ = ['i','g','i-g']\n",
    "filtered_ = False # whether to use filtered PAndAS data (filtered out values that aren't in 2MASS)\n",
    "predictions_list = []\n",
    "gc_filter = []\n",
    "\n",
    "print(f'Field {field}')\n",
    "for i in range(1):\n",
    "    predictions_list.append( rf_pred(field,training_data,crowding=250,n_estimators=n_trees,max_depth=max_depth_,filtered=filtered_,\n",
    "                                     max_leaf_nodes=max_leaf_nodes_,min_samples_leaf=min_samples_leaf_, features=features_) )\n",
    "print('Filtering...')\n",
    "for i in range(len(predictions_list[0])):\n",
    "    gc_candidate = all([ p[i]['pred']=='gc' for p in predictions_list ])\n",
    "    if gc_candidate: gc_filter.append(True)\n",
    "    else: gc_filter.append(False)\n",
    "gc_candidates = predictions_list[0][gc_filter]\n",
    "with open(f'pickle/predictions/1203/predictionsf{field}.pk','wb') as f:\n",
    "    pickle.dump(gc_candidates,f)\n",
    "print(len(gc_candidates))\n",
    "print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cc44004",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a396dda1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "9e0b3b6e",
   "metadata": {},
   "source": [
    "\n",
    "# Make Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "261a366a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# select most recent training data\n",
    "train_file = 'training_data_0802.pk' # training data with 3 classes\n",
    "train_file = 'training_data_1702.pk' # training data with only gcs and galaxies\n",
    "#train_file = 'training_data_1902_with_stars.pk' # training data with gcs galaxies and stars, classed as 'gc' and 'non-gc'\n",
    "#train_file = 'temp/train_plus_35.pk'\n",
    "#train_file = 'temp/train_plus_148.pk'\n",
    "#train_file = 'training_data_2702_j.pk'\n",
    "train_file = 'training_data_2704.pk'\n",
    "\n",
    "# load training data and filter out stars\n",
    "with open(f'./pickle/training_data/{train_file}','rb') as f:\n",
    "    training_data = pickle.load(f)\n",
    "\n",
    "#training_data = training_data[training_data['j_acc']==True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "64427163",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Field 35\n",
      "Filtering...\n",
      "1678\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "field = 35\n",
    "n_trees = 30\n",
    "max_depth_ = 9\n",
    "max_leaf_nodes_ = 12\n",
    "min_samples_leaf_ = 15\n",
    "features_ = ['i','g','i-g','j']\n",
    "features_ = ['i','g','g-i']\n",
    "predictions_list = []\n",
    "gc_filter = []\n",
    "\n",
    "print(f'Field {field}')\n",
    "for i in range(1):\n",
    "    predictions_list.append( rf_pred(field,training_data,crowding=250,n_estimators=n_trees,max_depth=max_depth_, max_leaf_nodes=max_leaf_nodes_,min_samples_leaf=min_samples_leaf_, features=features_) )\n",
    "print('Filtering...')\n",
    "for i in range(len(predictions_list[0])):\n",
    "    gc_candidate = all([ p[i]['pred']=='gc' for p in predictions_list ])\n",
    "    if gc_candidate: gc_filter.append(True)\n",
    "    else: gc_filter.append(False)\n",
    "gc_candidates = predictions_list[0][gc_filter]\n",
    "with open(f'pickle/predictions/predictionsf{field}.pk','wb') as f:\n",
    "    pickle.dump(gc_candidates,f)\n",
    "print(len(gc_candidates))\n",
    "print('\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a5ae2d1",
   "metadata": {},
   "source": [
    "## Make predictions from list of fields"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2008e4cb",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start:\n",
      "Field 135\n",
      "Filtering...\n",
      "620\n",
      "\n",
      "\n",
      "Field 148\n",
      "Filtering...\n",
      "2335\n",
      "\n",
      "\n",
      "Field 163\n",
      "Filtering...\n",
      "272\n",
      "\n",
      "\n",
      "Field 178\n",
      "Filtering...\n",
      "671\n",
      "\n",
      "\n",
      "Field 208\n",
      "Filtering...\n",
      "1735\n",
      "\n",
      "\n",
      "Field 207\n",
      "Filtering...\n",
      "1697\n",
      "\n",
      "\n",
      "Field 229\n",
      "Filtering...\n",
      "1587\n",
      "\n",
      "\n",
      "Field 222\n",
      "Filtering...\n",
      "2797\n",
      "\n",
      "\n",
      "Field 273\n",
      "Filtering...\n",
      "1020\n",
      "\n",
      "\n",
      "Field 274\n",
      "Filtering...\n",
      "974\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "field = 80\n",
    "\n",
    "#fields = [9,21,32,35,37,41,53,56,59,63,73,78,80,79,97,104,103,122,126,121,117,118,135]\n",
    "#fields = [34,35,36,53,56,59,63,78,80,103,101,99,148,146,85,88,86,5,162,135,188,186,184,185,169]\n",
    "fields = [135,148,163,178,208,207,229,222,273,274]\n",
    "\n",
    "date = '1303'\n",
    "n_trees = 40\n",
    "max_depth_ = 9\n",
    "max_leaf_nodes_ = 12\n",
    "min_samples_leaf_ = 15\n",
    "features_ = ['i','g','i-g']\n",
    "\n",
    "\n",
    "Path(f'./pickle/predictions/{date}').mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "\n",
    "predictions_list = []\n",
    "print('Start:')\n",
    "for field in fields:\n",
    "    predictions_list = []\n",
    "    gc_filter = []\n",
    "    print(f'Field {field}')\n",
    "    for i in range(1): # iterate to take the intersection of all predictions\n",
    "        predictions_list.append( rf_pred(field,training_data,n_estimators=n_trees,max_depth=max_depth_, max_leaf_nodes=max_leaf_nodes_,min_samples_leaf=min_samples_leaf_, features=features_) )\n",
    "    print('Filtering...')\n",
    "    for i in range(len(predictions_list[0])):\n",
    "        gc_candidate = all([ p[i]['pred']=='gc' for p in predictions_list ]) # select only gcs that were predicted on all iterations\n",
    "        if gc_candidate: gc_filter.append(True)\n",
    "        else: gc_filter.append(False)\n",
    "    gc_candidates = predictions_list[0][gc_filter]\n",
    "    with open(f'pickle/predictions/{date}/predictionsf{field}.pk','wb') as f:\n",
    "        pickle.dump(gc_candidates,f)\n",
    "    print(len(gc_candidates))\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "id": "d66a5e24",
   "metadata": {},
   "outputs": [],
   "source": [
    "gc_filter = []\n",
    "for i in range(len(predictions_list[0])):\n",
    "    gc_candidate = all([ p[i]['pred']=='gc' for p in predictions_list ])\n",
    "    if gc_candidate: gc_filter.append(True)\n",
    "    else: gc_filter.append(False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "id": "7aa9d870",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "gc_candidates = predictions_list[0][gc_filter]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "f5c2895c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "150"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save predictions to pickle file and print how many GCs were found\n",
    "with open(f'pickle/predictionsf{field}.pk','wb') as f:\n",
    "    pickle.dump(gc_candidates,f)\n",
    "len(gc_candidates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "767fe7df",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}