Final_delivery.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/media/alessandro/storage/anaconda3/lib/python3.6/site-packages/sklearn/ensemble/weight_boosting.py:29: DeprecationWarning: numpy.core.umath_tests is an internal NumPy module and should not be imported. It will be removed in a future NumPy release.\n",
      "  from numpy.core.umath_tests import inner1d\n"
     ]
    }
   ],
   "source": [
    "# imports for array-handling and plotting and time\n",
    "import numpy as np\n",
    "import time\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# keras imports for the dataset and building neural network\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.layers import Dense, Dropout, Activation, Conv2D, Flatten, MaxPooling2D\n",
    "from keras.utils import np_utils, to_categorical\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "\n",
    "# sklearn imports for model selection, data preparation and classifiers\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedKFold, cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.decomposition import IncrementalPCA\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "images = np.loadtxt(\"handwritten_digits_images.csv\", delimiter=',')\n",
    "labels = np.loadtxt(\"handwritten_digits_labels.csv\", delimiter=',')\n",
    "\n",
    "# Making sure that the values are float so that we can get decimal points after division\n",
    "images = images.astype('float32')\n",
    "\n",
    "# Normalizing the RGB codes by dividing it to the max RGB value.\n",
    "images /= 255\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(images, labels,test_size=0.3, random_state=32)\n",
    "\n",
    "#When we are not wrapping our nn into a KerasClassifier we need hot encoded y \n",
    "y_train_enc = np_utils.to_categorical(y_train, 10)\n",
    "y_test_enc = np_utils.to_categorical(y_test, 10)\n",
    "\n",
    "images1 = images[1::4]\n",
    "labels1 = labels[1::4]\n",
    "\n",
    "pca = IncrementalPCA(n_components=40, batch_size=100)\n",
    "images_pca = pca.fit_transform(images1)\n",
    "\n",
    "X_train3, X_test3, y_train3, y_test3 = train_test_split(images_pca, labels1,test_size=0.3, random_state=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "---------------------- 1HiddenLayer NN --------------------------------------\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'batch_size': 50, 'nb_epoch': 20}\n",
      "\n",
      "Train on 49000 samples, validate on 21000 samples\n",
      "Epoch 1/15\n",
      " - 5s - loss: 0.2508 - acc: 0.9270 - val_loss: 0.1417 - val_acc: 0.9597\n",
      "Epoch 2/15\n",
      " - 5s - loss: 0.1098 - acc: 0.9669 - val_loss: 0.1038 - val_acc: 0.9683\n",
      "Epoch 3/15\n",
      " - 5s - loss: 0.0766 - acc: 0.9766 - val_loss: 0.0828 - val_acc: 0.9750\n",
      "Epoch 4/15\n",
      " - 5s - loss: 0.0585 - acc: 0.9816 - val_loss: 0.0890 - val_acc: 0.9733\n",
      "Epoch 5/15\n",
      " - 5s - loss: 0.0442 - acc: 0.9860 - val_loss: 0.0756 - val_acc: 0.9778\n",
      "Epoch 6/15\n",
      " - 5s - loss: 0.0360 - acc: 0.9883 - val_loss: 0.0812 - val_acc: 0.9755\n",
      "Epoch 7/15\n",
      " - 5s - loss: 0.0311 - acc: 0.9900 - val_loss: 0.0740 - val_acc: 0.9780\n",
      "Epoch 8/15\n",
      " - 5s - loss: 0.0260 - acc: 0.9918 - val_loss: 0.0722 - val_acc: 0.9789\n",
      "Epoch 9/15\n",
      " - 5s - loss: 0.0252 - acc: 0.9914 - val_loss: 0.0745 - val_acc: 0.9805\n",
      "Epoch 10/15\n",
      " - 5s - loss: 0.0192 - acc: 0.9939 - val_loss: 0.0792 - val_acc: 0.9789\n",
      "Epoch 11/15\n",
      " - 5s - loss: 0.0181 - acc: 0.9939 - val_loss: 0.0734 - val_acc: 0.9817\n",
      "Epoch 12/15\n",
      " - 4s - loss: 0.0162 - acc: 0.9946 - val_loss: 0.0772 - val_acc: 0.9813\n",
      "Epoch 13/15\n",
      " - 5s - loss: 0.0157 - acc: 0.9946 - val_loss: 0.0839 - val_acc: 0.9792\n",
      "Epoch 14/15\n",
      " - 5s - loss: 0.0143 - acc: 0.9950 - val_loss: 0.0900 - val_acc: 0.9786\n",
      "Epoch 15/15\n",
      " - 5s - loss: 0.0120 - acc: 0.9959 - val_loss: 0.0862 - val_acc: 0.9800\n",
      "\n",
      "Test Loss :  0.0861983475477443\n",
      "Test Accuracy :  98.0 %\n",
      "Cross Validation Time :  113.53 sec\n",
      "Training Time :  68.35 sec\n"
     ]
    }
   ],
   "source": [
    "print()\n",
    "print(\"---------------------- 1HiddenLayer NN --------------------------------------\")\n",
    "print()\n",
    "\n",
    "def make_model1():\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Dense(512, input_shape=(784,)))\n",
    "    model.add(Activation('relu'))                            \n",
    "    model.add(Dropout(0.2))\n",
    "    model.add(Dense(10))\n",
    "    model.add(Activation('softmax'))\n",
    "    model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')    \n",
    "    \n",
    "    return model\n",
    "\n",
    "\n",
    "def tuning1(X_train,Y_train,X_test,Y_test):\n",
    "\n",
    "    batch_size = [50, 80, 100, 128, 135, 150]\n",
    "    epochs = [15,20,25]\n",
    "    param_grid = dict(batch_size=batch_size, nb_epoch=epochs)\n",
    "\n",
    "    k_model = KerasClassifier(build_fn=make_model1, verbose=0)\n",
    "   \n",
    "    clf = GridSearchCV(estimator=k_model, param_grid=param_grid, \n",
    "                                   cv=3,\n",
    "                                   scoring=\"accuracy\", verbose=0 ,n_jobs=-1)\n",
    "    clf.fit(X_train,Y_train)\n",
    "    \n",
    "    print(\"Best parameters set found on development set:\")\n",
    "    print()\n",
    "    print(clf.best_params_)\n",
    "    \n",
    "    model=make_model1()\n",
    "    \n",
    "    return clf,model\n",
    "  \n",
    "tuning1_start = time.time()    \n",
    "clf1 = tuning1(X_train,y_train,X_test,y_test)\n",
    "tuning1_end = time.time()\n",
    "print()\n",
    "model1 = clf1[1]\n",
    "\n",
    "fit1_start=time.time()\n",
    "history1 = model1.fit(X_train,y_train_enc, batch_size=50, epochs=15 ,verbose=2,validation_data=(X_test, y_test_enc))\n",
    "fit1_end=time.time()\n",
    "\n",
    "#Evaluating the model\n",
    "loss1, accuracy1 = model1.evaluate(X_test, y_test_enc, verbose=2)\n",
    "\n",
    "print()\n",
    "print(\"Test Loss : \", loss1)\n",
    "print(\"Test Accuracy : \", round(accuracy1*100, 2), \"%\")\n",
    "print(\"Cross Validation Time : \", round(tuning1_end-tuning1_start, 2), \"sec\" )\n",
    "print(\"Training Time : \",  round(fit1_end-fit1_start, 2), \"sec\"  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Taken from the history of the Keras' repo.\n",
    "#Since Keras 2.0 metrics fmeasure, precision, and recall have been removed.\n",
    "\n",
    " \n",
    "from keras import backend as K\n",
    "\n",
    "def precision(y_true, y_pred):\n",
    "    \"\"\"Precision metric.\n",
    "\n",
    "    Only computes a batch-wise average of precision.\n",
    "\n",
    "    Computes the precision, a metric for multi-label classification of\n",
    "    how many selected items are relevant.\n",
    "    \"\"\"\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "    precision = true_positives / (predicted_positives + K.epsilon())\n",
    "    return precision\n",
    "\n",
    "def recall(y_true, y_pred):\n",
    "    \"\"\"Recall metric.\n",
    "\n",
    "    Only computes a batch-wise average of recall.\n",
    "\n",
    "    Computes the recall, a metric for multi-label classification of\n",
    "    how many relevant items are selected.\n",
    "    \"\"\"\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "    recall = true_positives / (possible_positives + K.epsilon())\n",
    "    return recall\n",
    "\n",
    "\n",
    "def f1(y_true, y_pred):\n",
    "    def recall(y_true, y_pred):\n",
    "        \"\"\"Recall metric.\n",
    "\n",
    "        Only computes a batch-wise average of recall.\n",
    "\n",
    "        Computes the recall, a metric for multi-label classification of\n",
    "        how many relevant items are selected.\n",
    "        \"\"\"\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "        recall = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall\n",
    "\n",
    "    def precision(y_true, y_pred):\n",
    "        \"\"\"Precision metric.\n",
    "\n",
    "        Only computes a batch-wise average of precision.\n",
    "\n",
    "        Computes the precision, a metric for multi-label classification of\n",
    "        how many selected items are relevant.\n",
    "        \"\"\"\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "        precision = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision\n",
    "    precision = precision(y_true, y_pred)\n",
    "    recall = recall(y_true, y_pred)\n",
    "    return 2*((precision*recall)/(precision+recall+K.epsilon()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "---------------------- Convolutional NN --------------------------------------\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'nb_epoch': 3}\n",
      "\n",
      "Train on 49000 samples, validate on 21000 samples\n",
      "Epoch 1/3\n",
      "49000/49000 [==============================] - 16s 327us/step - loss: 0.2722 - acc: 0.9164 - f1: 0.9019 - precision: 0.9280 - recall: 0.8853 - val_loss: 0.0983 - val_acc: 0.9710 - val_f1: 0.9712 - val_precision: 0.9760 - val_recall: 0.9666\n",
      "Epoch 2/3\n",
      "49000/49000 [==============================] - 16s 322us/step - loss: 0.0985 - acc: 0.9709 - f1: 0.9712 - precision: 0.9756 - recall: 0.9669 - val_loss: 0.0665 - val_acc: 0.9800 - val_f1: 0.9801 - val_precision: 0.9827 - val_recall: 0.9776\n",
      "Epoch 3/3\n",
      "49000/49000 [==============================] - 16s 322us/step - loss: 0.0738 - acc: 0.9776 - f1: 0.9779 - precision: 0.9809 - recall: 0.9751 - val_loss: 0.0623 - val_acc: 0.9819 - val_f1: 0.9822 - val_precision: 0.9840 - val_recall: 0.9805\n",
      "\n",
      "Test Loss :  0.062291610156230276\n",
      "Test Accuracy :  98.19 %\n",
      "F1-Measure 0.9822\n",
      "Precision 0.984\n",
      "Recall 0.9805\n",
      "Cross Validation Time :  227.55 sec\n",
      "Training Time :  47.86 sec\n",
      "\n",
      "Confusion Matrix : \n",
      "\n",
      "[[2011    1    8    2    1    8    4    0    3    0]\n",
      " [   1 2338   25    5    3    2    2    6    6    0]\n",
      " [   0    0 2032   14    0    0    1   13    0    1]\n",
      " [   0    0   11 2131    0    5    1   10    1    3]\n",
      " [   3    2    7    0 1992    0    9    8    3   32]\n",
      " [   2    0    1   16    0 1854    4    1    1    1]\n",
      " [   5    1    0    0    2    4 2056    0    6    0]\n",
      " [   2    1    8    6    1    1    0 2141    2    3]\n",
      " [   2    1   13   11    3   17    2    3 1989    9]\n",
      " [   4    0    2   14    3   11    2   12    3 2075]]\n",
      "\n",
      "\n",
      "20619  classified correctly\n",
      "381  classified incorrectly\n",
      "\n",
      "Error rate :  1.81 %\n"
     ]
    }
   ],
   "source": [
    "print()\n",
    "print(\"---------------------- Convolutional NN --------------------------------------\")\n",
    "print()\n",
    "\n",
    "#reshape data to fit model\n",
    "X_train_conv = X_train.reshape(49000,28,28,1)\n",
    "X_test_conv = X_test.reshape(21000,28,28,1)\n",
    "\n",
    "def make_model2():\n",
    "    \n",
    "    model = Sequential()\n",
    "\n",
    "    model.add(Conv2D(10,kernel_size=5, activation='relu', input_shape=(28,28,1)))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Conv2D(20, kernel_size=5, activation='relu'))\n",
    "    model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "    model.add(Dropout(0.25))\n",
    "\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(10, activation='softmax'))\n",
    "\n",
    "    model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy', f1, precision, recall])  \n",
    "    \n",
    "    return model\n",
    "\n",
    "\n",
    "\n",
    "def tuning2(X_train,Y_train,X_test,Y_test):\n",
    "\n",
    "    \n",
    "    epochs = list(range(1,5))\n",
    "    param_grid = dict(nb_epoch=epochs)\n",
    "\n",
    "    k_model = KerasClassifier(build_fn=make_model2, verbose=0)\n",
    "   \n",
    "    clf = GridSearchCV(estimator=k_model, param_grid=param_grid, \n",
    "                                   cv=5,\n",
    "                                   scoring=\"accuracy\", verbose=0 ,n_jobs=-1)\n",
    "    clf.fit(X_train,Y_train)\n",
    "    \n",
    "    print(\"Best parameters set found on development set:\")\n",
    "    print()\n",
    "    print(clf.best_params_)\n",
    "    \n",
    "    model=make_model2()\n",
    "    \n",
    "    return clf,model\n",
    "    \n",
    "tuning2_start = time.time()    \n",
    "clf2 = tuning2(X_train_conv,y_train,X_test_conv,y_test)\n",
    "tuning2_end = time.time()\n",
    "tuning_time = tuning2_end-tuning2_start\n",
    "print()\n",
    "model2 = clf2[1]\n",
    "\n",
    "fit2_start=time.time()\n",
    "model2.fit(X_train_conv, y_train_enc, validation_data=(X_test_conv, y_test_enc), epochs=3)\n",
    "fit2_end=time.time()\n",
    "exec_time = fit2_end-fit2_start\n",
    "\n",
    "#Evaluating the model\n",
    "evaluation = model2.evaluate(X_test_conv, y_test_enc, verbose=2)\n",
    "\n",
    "loss = evaluation[0]\n",
    "accuracy = evaluation[1]\n",
    "f1_measure = evaluation[2]\n",
    "precision = evaluation[3]\n",
    "recall = evaluation[4]\n",
    "\n",
    "print()\n",
    "print(\"Test Loss : \", loss)\n",
    "print(\"Test Accuracy : \", round(accuracy*100, 2), \"%\")\n",
    "print(\"F1-Measure\", round(f1_measure, 4))\n",
    "print(\"Precision\", round(precision, 4))\n",
    "print(\"Recall\", round(recall, 4))\n",
    "print(\"Cross Validation Time : \", round(tuning_time, 2), \"sec\" )\n",
    "print(\"Training Time : \",  round(exec_time, 2), \"sec\"  )\n",
    "\n",
    "\n",
    "predicted_classes = model2.predict_classes(X_test_conv)\n",
    "conmat = confusion_matrix(y_test, predicted_classes)\n",
    "\n",
    "print()\n",
    "print('Confusion Matrix : ')\n",
    "print()\n",
    "print(conmat)\n",
    "print()\n",
    "\n",
    "# see which we predicted correctly and which not ----- Indices of elements that are non-zero.\n",
    "correct_indices = np.nonzero(predicted_classes == y_test)[0]\n",
    "incorrect_indices = np.nonzero(predicted_classes != y_test)[0]\n",
    "\n",
    "print()\n",
    "print(len(correct_indices),\" classified correctly\")\n",
    "print(len(incorrect_indices),\" classified incorrectly\")\n",
    "\n",
    "error_rate = len(incorrect_indices)/(len(correct_indices)+len(incorrect_indices))\n",
    "\n",
    "print()\n",
    "print(\"Error rate : \", round(error_rate*100, 2), \"%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "---------------------- K-NN Classifier --------------------------------------\n",
      "\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'n_neighbors': 3}\n",
      "\n",
      "Test Accuracy :  96.34 %\n",
      "Cross Validation Time :  0.0 sec\n",
      "Training Time :  42.37 sec\n"
     ]
    }
   ],
   "source": [
    "print()\n",
    "print(\"---------------------- K-NN Classifier --------------------------------------\")\n",
    "print()\n",
    "\n",
    "KnnClassifier=KNeighborsClassifier()\n",
    "\n",
    "grid_params = {\"n_neighbors\": range(1, 10)}\n",
    "\n",
    "tuning3_start = time.time()    \n",
    "grid_search = GridSearchCV(KnnClassifier, grid_params, verbose=0, n_jobs=-1)\n",
    "tuning3_end = time.time()\n",
    "\n",
    "fit3_start=time.time()\n",
    "grid_search.fit(X_train3, y_train3)\n",
    "fit3_end=time.time()\n",
    "\n",
    "predicted = grid_search.predict(X_test3)\n",
    "acc = accuracy_score(y_test3, predicted)\n",
    "\n",
    "print()\n",
    "print(\"Best parameters set found on development set:\")\n",
    "print()\n",
    "print(grid_search.best_params_)\n",
    "\n",
    "print()\n",
    "print(\"Test Accuracy : \", round(acc*100, 2), \"%\")\n",
    "print(\"Cross Validation Time : \", round(tuning3_end-tuning3_start, 2), \"sec\" )\n",
    "print(\"Training Time : \",  round(fit3_end-fit3_start, 2), \"sec\"  )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "---------------------- RandomForest Classifier --------------------------------------\n",
      "\n",
      "\n",
      "Best parameters set found on development set:\n",
      "\n",
      "{'n_estimators': 19}\n",
      "\n",
      "Test Accuracy :  95.67 %\n",
      "Cross Validation Time :  0.0 sec\n",
      "Training Time :  36.1 sec\n"
     ]
    }
   ],
   "source": [
    "print()\n",
    "print(\"---------------------- RandomForest Classifier --------------------------------------\")\n",
    "print()\n",
    "\n",
    "rfc = RandomForestClassifier(n_jobs=-1)\n",
    "\n",
    "grid_params1 = {\"n_estimators\": range(1, 20)}\n",
    "\n",
    "tuning4_start = time.time()    \n",
    "grid_search1 = GridSearchCV(rfc, grid_params1, verbose=0, n_jobs=-1)\n",
    "tuning4_end = time.time()\n",
    "\n",
    "fit4_start=time.time()\n",
    "grid_search1.fit(X_train, y_train)\n",
    "fit4_end=time.time()\n",
    "\n",
    "predicted1 = grid_search1.predict(X_test)\n",
    "acc1 = accuracy_score(y_test, predicted1)\n",
    "\n",
    "print()\n",
    "print(\"Best parameters set found on development set:\")\n",
    "print()\n",
    "print(grid_search1.best_params_)\n",
    "\n",
    "print()\n",
    "print(\"Test Accuracy : \", round(acc1*100, 2), \"%\")\n",
    "print(\"Cross Validation Time : \", round(tuning4_end-tuning4_start, 2), \"sec\" )\n",
    "print(\"Training Time : \",  round(fit4_end-fit4_start, 2), \"sec\"  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature ranking:\n",
      "1. feature 433 (0.011760)\n",
      "2. feature 378 (0.010729)\n",
      "3. feature 542 (0.009558)\n",
      "4. feature 350 (0.009426)\n",
      "5. feature 483 (0.008373)\n",
      "6. feature 377 (0.007779)\n",
      "7. feature 434 (0.007545)\n",
      "8. feature 211 (0.007334)\n",
      "9. feature 517 (0.007316)\n",
      "10. feature 461 (0.007304)\n",
      "11. feature 515 (0.007208)\n",
      "12. feature 351 (0.007084)\n",
      "13. feature 210 (0.006836)\n",
      "14. feature 569 (0.006727)\n",
      "15. feature 381 (0.006589)\n",
      "16. feature 262 (0.006499)\n",
      "17. feature 153 (0.006455)\n",
      "18. feature 488 (0.006391)\n",
      "19. feature 346 (0.006250)\n",
      "20. feature 291 (0.006194)\n",
      "21. feature 429 (0.006176)\n",
      "22. feature 156 (0.006132)\n",
      "23. feature 375 (0.006009)\n",
      "24. feature 183 (0.005926)\n",
      "25. feature 354 (0.005916)\n",
      "26. feature 486 (0.005837)\n",
      "27. feature 624 (0.005676)\n",
      "28. feature 297 (0.005670)\n",
      "29. feature 490 (0.005540)\n",
      "30. feature 430 (0.005525)\n",
      "31. feature 657 (0.005433)\n",
      "32. feature 572 (0.005421)\n",
      "33. feature 464 (0.005406)\n",
      "34. feature 348 (0.005364)\n",
      "35. feature 318 (0.005343)\n",
      "36. feature 290 (0.005320)\n",
      "37. feature 410 (0.005272)\n",
      "38. feature 270 (0.005214)\n",
      "39. feature 238 (0.005206)\n",
      "40. feature 347 (0.005180)\n",
      "41. feature 570 (0.005172)\n",
      "42. feature 382 (0.005093)\n",
      "43. feature 462 (0.005026)\n",
      "44. feature 152 (0.005002)\n",
      "45. feature 379 (0.004929)\n",
      "46. feature 239 (0.004871)\n",
      "47. feature 514 (0.004816)\n",
      "48. feature 323 (0.004773)\n",
      "49. feature 656 (0.004730)\n",
      "50. feature 456 (0.004700)\n",
      "51. feature 513 (0.004696)\n",
      "52. feature 349 (0.004677)\n",
      "53. feature 459 (0.004665)\n",
      "54. feature 568 (0.004632)\n",
      "55. feature 155 (0.004631)\n",
      "56. feature 402 (0.004622)\n",
      "57. feature 438 (0.004607)\n",
      "58. feature 596 (0.004551)\n",
      "59. feature 458 (0.004529)\n",
      "60. feature 405 (0.004527)\n",
      "61. feature 460 (0.004505)\n",
      "62. feature 437 (0.004497)\n",
      "63. feature 404 (0.004470)\n",
      "64. feature 406 (0.004454)\n",
      "65. feature 432 (0.004411)\n",
      "66. feature 266 (0.004364)\n",
      "67. feature 435 (0.004306)\n",
      "68. feature 545 (0.004239)\n",
      "69. feature 236 (0.004234)\n",
      "70. feature 326 (0.004199)\n",
      "71. feature 237 (0.004032)\n",
      "72. feature 184 (0.004019)\n",
      "73. feature 154 (0.003970)\n",
      "74. feature 543 (0.003969)\n",
      "75. feature 403 (0.003956)\n",
      "76. feature 267 (0.003917)\n",
      "77. feature 299 (0.003912)\n",
      "78. feature 463 (0.003893)\n",
      "79. feature 240 (0.003892)\n",
      "80. feature 544 (0.003879)\n",
      "81. feature 292 (0.003834)\n",
      "82. feature 400 (0.003821)\n",
      "83. feature 353 (0.003794)\n",
      "84. feature 182 (0.003760)\n",
      "85. feature 295 (0.003758)\n",
      "86. feature 376 (0.003749)\n",
      "87. feature 268 (0.003746)\n",
      "88. feature 489 (0.003726)\n",
      "89. feature 541 (0.003711)\n",
      "90. feature 374 (0.003709)\n",
      "91. feature 516 (0.003701)\n",
      "92. feature 325 (0.003695)\n",
      "93. feature 320 (0.003616)\n",
      "94. feature 358 (0.003603)\n",
      "95. feature 322 (0.003592)\n",
      "96. feature 329 (0.003589)\n",
      "97. feature 380 (0.003576)\n",
      "98. feature 373 (0.003573)\n",
      "99. feature 324 (0.003556)\n",
      "100. feature 512 (0.003523)\n",
      "101. feature 487 (0.003515)\n",
      "102. feature 401 (0.003508)\n",
      "103. feature 317 (0.003495)\n",
      "104. feature 212 (0.003485)\n",
      "105. feature 485 (0.003482)\n",
      "106. feature 540 (0.003462)\n",
      "107. feature 319 (0.003460)\n",
      "108. feature 296 (0.003443)\n",
      "109. feature 264 (0.003383)\n",
      "110. feature 550 (0.003372)\n",
      "111. feature 428 (0.003347)\n",
      "112. feature 352 (0.003344)\n",
      "113. feature 269 (0.003344)\n",
      "114. feature 658 (0.003320)\n",
      "115. feature 345 (0.003312)\n",
      "116. feature 484 (0.003254)\n",
      "117. feature 455 (0.003241)\n",
      "118. feature 625 (0.003237)\n",
      "119. feature 273 (0.003183)\n",
      "120. feature 457 (0.003180)\n",
      "121. feature 344 (0.003179)\n",
      "122. feature 712 (0.003155)\n",
      "123. feature 209 (0.003145)\n",
      "124. feature 289 (0.003121)\n",
      "125. feature 413 (0.003117)\n",
      "126. feature 626 (0.003114)\n",
      "127. feature 215 (0.003106)\n",
      "128. feature 431 (0.003100)\n",
      "129. feature 294 (0.003013)\n",
      "130. feature 573 (0.003007)\n",
      "131. feature 571 (0.002999)\n",
      "132. feature 316 (0.002990)\n",
      "133. feature 293 (0.002956)\n",
      "134. feature 271 (0.002956)\n",
      "135. feature 654 (0.002955)\n",
      "136. feature 522 (0.002954)\n",
      "137. feature 298 (0.002919)\n",
      "138. feature 321 (0.002887)\n",
      "139. feature 235 (0.002887)\n",
      "140. feature 627 (0.002885)\n",
      "141. feature 598 (0.002878)\n",
      "142. feature 436 (0.002868)\n",
      "143. feature 214 (0.002839)\n",
      "144. feature 272 (0.002803)\n",
      "145. feature 655 (0.002768)\n",
      "146. feature 372 (0.002757)\n",
      "147. feature 427 (0.002755)\n",
      "148. feature 181 (0.002722)\n",
      "149. feature 493 (0.002709)\n",
      "150. feature 518 (0.002685)\n",
      "151. feature 244 (0.002673)\n",
      "152. feature 399 (0.002662)\n",
      "153. feature 263 (0.002657)\n",
      "154. feature 577 (0.002630)\n",
      "155. feature 207 (0.002626)\n",
      "156. feature 265 (0.002611)\n",
      "157. feature 566 (0.002604)\n",
      "158. feature 523 (0.002602)\n",
      "159. feature 491 (0.002588)\n",
      "160. feature 597 (0.002570)\n",
      "161. feature 371 (0.002565)\n",
      "162. feature 409 (0.002543)\n",
      "163. feature 659 (0.002514)\n",
      "164. feature 495 (0.002479)\n",
      "165. feature 206 (0.002473)\n",
      "166. feature 213 (0.002453)\n",
      "167. feature 241 (0.002398)\n",
      "168. feature 185 (0.002392)\n",
      "169. feature 466 (0.002382)\n",
      "170. feature 408 (0.002366)\n",
      "171. feature 470 (0.002365)\n",
      "172. feature 178 (0.002339)\n",
      "173. feature 208 (0.002327)\n",
      "174. feature 242 (0.002319)\n",
      "175. feature 99 (0.002317)\n",
      "176. feature 492 (0.002315)\n",
      "177. feature 521 (0.002273)\n",
      "178. feature 355 (0.002237)\n",
      "179. feature 575 (0.002221)\n",
      "180. feature 407 (0.002214)\n",
      "181. feature 180 (0.002181)\n",
      "182. feature 567 (0.002178)\n",
      "183. feature 630 (0.002170)\n",
      "184. feature 300 (0.002138)\n",
      "185. feature 127 (0.002113)\n",
      "186. feature 414 (0.002106)\n",
      "187. feature 426 (0.002050)\n",
      "188. feature 327 (0.002002)\n",
      "189. feature 186 (0.002002)\n",
      "190. feature 328 (0.001969)\n",
      "191. feature 441 (0.001964)\n",
      "192. feature 234 (0.001960)\n",
      "193. feature 233 (0.001952)\n",
      "194. feature 465 (0.001943)\n",
      "195. feature 124 (0.001932)\n",
      "196. feature 546 (0.001922)\n",
      "197. feature 539 (0.001917)\n",
      "198. feature 482 (0.001909)\n",
      "199. feature 243 (0.001896)\n",
      "200. feature 467 (0.001891)\n",
      "201. feature 595 (0.001878)\n",
      "202. feature 524 (0.001877)\n",
      "203. feature 632 (0.001859)\n",
      "204. feature 102 (0.001835)\n",
      "205. feature 496 (0.001823)\n",
      "206. feature 288 (0.001822)\n",
      "207. feature 574 (0.001808)\n",
      "208. feature 157 (0.001776)\n",
      "209. feature 653 (0.001770)\n",
      "210. feature 204 (0.001758)\n",
      "211. feature 151 (0.001754)\n",
      "212. feature 384 (0.001754)\n",
      "213. feature 549 (0.001754)\n",
      "214. feature 511 (0.001735)\n",
      "215. feature 629 (0.001733)\n",
      "216. feature 633 (0.001706)\n",
      "217. feature 343 (0.001692)\n",
      "218. feature 628 (0.001692)\n",
      "219. feature 494 (0.001692)\n",
      "220. feature 261 (0.001668)\n",
      "221. feature 600 (0.001665)\n",
      "222. feature 548 (0.001619)\n",
      "223. feature 581 (0.001604)\n",
      "224. feature 602 (0.001602)\n",
      "225. feature 520 (0.001602)\n",
      "226. feature 547 (0.001600)\n",
      "227. feature 125 (0.001597)\n",
      "228. feature 601 (0.001593)\n",
      "229. feature 150 (0.001568)\n",
      "230. feature 519 (0.001566)\n",
      "231. feature 576 (0.001560)\n",
      "232. feature 356 (0.001557)\n",
      "233. feature 411 (0.001555)\n",
      "234. feature 439 (0.001554)\n",
      "235. feature 386 (0.001522)\n",
      "236. feature 714 (0.001516)\n",
      "237. feature 179 (0.001514)\n",
      "238. feature 159 (0.001511)\n",
      "239. feature 190 (0.001508)\n",
      "240. feature 128 (0.001492)\n",
      "241. feature 469 (0.001485)\n",
      "242. feature 189 (0.001484)\n",
      "243. feature 160 (0.001483)\n",
      "244. feature 246 (0.001473)\n",
      "245. feature 232 (0.001450)\n",
      "246. feature 599 (0.001447)\n",
      "247. feature 217 (0.001442)\n",
      "248. feature 497 (0.001424)\n",
      "249. feature 126 (0.001421)\n",
      "250. feature 537 (0.001415)\n",
      "251. feature 468 (0.001412)\n",
      "252. feature 301 (0.001407)\n",
      "253. feature 383 (0.001400)\n",
      "254. feature 551 (0.001392)\n",
      "255. feature 177 (0.001368)\n",
      "256. feature 660 (0.001365)\n",
      "257. feature 565 (0.001364)\n",
      "258. feature 603 (0.001363)\n",
      "259. feature 552 (0.001347)\n",
      "260. feature 579 (0.001336)\n",
      "261. feature 130 (0.001332)\n",
      "262. feature 685 (0.001330)\n",
      "263. feature 187 (0.001318)\n",
      "264. feature 158 (0.001317)\n",
      "265. feature 578 (0.001313)\n",
      "266. feature 604 (0.001285)\n",
      "267. feature 631 (0.001282)\n",
      "268. feature 606 (0.001263)\n",
      "269. feature 538 (0.001254)\n",
      "270. feature 605 (0.001249)\n",
      "271. feature 216 (0.001236)\n",
      "272. feature 205 (0.001232)\n",
      "273. feature 440 (0.001227)\n",
      "274. feature 357 (0.001219)\n",
      "275. feature 258 (0.001218)\n",
      "276. feature 623 (0.001216)\n",
      "277. feature 454 (0.001211)\n",
      "278. feature 218 (0.001209)\n",
      "279. feature 684 (0.001199)\n",
      "280. feature 330 (0.001198)\n",
      "281. feature 191 (0.001198)\n",
      "282. feature 274 (0.001187)\n",
      "283. feature 652 (0.001187)\n",
      "284. feature 510 (0.001173)\n",
      "285. feature 245 (0.001123)\n",
      "286. feature 101 (0.001119)\n",
      "287. feature 260 (0.001106)\n",
      "288. feature 412 (0.001100)\n",
      "289. feature 315 (0.001096)\n",
      "290. feature 188 (0.001070)\n",
      "291. feature 247 (0.001065)\n",
      "292. feature 453 (0.001061)\n",
      "293. feature 129 (0.001031)\n",
      "294. feature 683 (0.001016)\n",
      "295. feature 342 (0.001000)\n",
      "296. feature 661 (0.000988)\n",
      "297. feature 608 (0.000985)\n",
      "298. feature 287 (0.000977)\n",
      "299. feature 553 (0.000975)\n",
      "300. feature 634 (0.000971)\n",
      "301. feature 149 (0.000966)\n",
      "302. feature 398 (0.000941)\n",
      "303. feature 161 (0.000941)\n",
      "304. feature 686 (0.000936)\n",
      "305. feature 98 (0.000928)\n",
      "306. feature 385 (0.000925)\n",
      "307. feature 259 (0.000912)\n",
      "308. feature 607 (0.000905)\n",
      "309. feature 123 (0.000902)\n",
      "310. feature 303 (0.000898)\n",
      "311. feature 219 (0.000887)\n",
      "312. feature 370 (0.000873)\n",
      "313. feature 231 (0.000866)\n",
      "314. feature 525 (0.000863)\n",
      "315. feature 682 (0.000853)\n",
      "316. feature 498 (0.000824)\n",
      "317. feature 635 (0.000820)\n",
      "318. feature 711 (0.000820)\n",
      "319. feature 302 (0.000803)\n",
      "320. feature 687 (0.000788)\n",
      "321. feature 594 (0.000781)\n",
      "322. feature 175 (0.000779)\n",
      "323. feature 580 (0.000776)\n",
      "324. feature 97 (0.000772)\n",
      "325. feature 176 (0.000760)\n",
      "326. feature 681 (0.000737)\n",
      "327. feature 609 (0.000735)\n",
      "328. feature 680 (0.000717)\n",
      "329. feature 651 (0.000708)\n",
      "330. feature 95 (0.000704)\n",
      "331. feature 275 (0.000702)\n",
      "332. feature 100 (0.000696)\n",
      "333. feature 148 (0.000690)\n",
      "334. feature 509 (0.000690)\n",
      "335. feature 314 (0.000678)\n",
      "336. feature 717 (0.000669)\n",
      "337. feature 710 (0.000665)\n",
      "338. feature 688 (0.000648)\n",
      "339. feature 286 (0.000646)\n",
      "340. feature 526 (0.000643)\n",
      "341. feature 331 (0.000629)\n",
      "342. feature 442 (0.000628)\n",
      "343. feature 203 (0.000626)\n",
      "344. feature 230 (0.000625)\n",
      "345. feature 554 (0.000613)\n",
      "346. feature 662 (0.000607)\n",
      "347. feature 715 (0.000598)\n",
      "348. feature 622 (0.000593)\n",
      "349. feature 248 (0.000588)\n",
      "350. feature 162 (0.000578)\n",
      "351. feature 709 (0.000557)\n",
      "352. feature 131 (0.000552)\n",
      "353. feature 718 (0.000547)\n",
      "354. feature 713 (0.000531)\n",
      "355. feature 443 (0.000494)\n",
      "356. feature 636 (0.000486)\n",
      "357. feature 121 (0.000482)\n",
      "358. feature 220 (0.000481)\n",
      "359. feature 276 (0.000472)\n",
      "360. feature 359 (0.000470)\n",
      "361. feature 103 (0.000468)\n",
      "362. feature 708 (0.000468)\n",
      "363. feature 689 (0.000467)\n",
      "364. feature 679 (0.000464)\n",
      "365. feature 481 (0.000458)\n",
      "366. feature 415 (0.000452)\n",
      "367. feature 96 (0.000436)\n",
      "368. feature 122 (0.000436)\n",
      "369. feature 637 (0.000435)\n",
      "370. feature 582 (0.000415)\n",
      "371. feature 555 (0.000413)\n",
      "372. feature 387 (0.000402)\n",
      "373. feature 132 (0.000396)\n",
      "374. feature 499 (0.000378)\n",
      "375. feature 663 (0.000378)\n",
      "376. feature 690 (0.000375)\n",
      "377. feature 471 (0.000374)\n",
      "378. feature 527 (0.000368)\n",
      "379. feature 192 (0.000357)\n",
      "380. feature 369 (0.000352)\n",
      "381. feature 425 (0.000345)\n",
      "382. feature 304 (0.000333)\n",
      "383. feature 593 (0.000333)\n",
      "384. feature 707 (0.000331)\n",
      "385. feature 341 (0.000324)\n",
      "386. feature 650 (0.000319)\n",
      "387. feature 719 (0.000318)\n",
      "388. feature 163 (0.000315)\n",
      "389. feature 202 (0.000313)\n",
      "390. feature 678 (0.000312)\n",
      "391. feature 397 (0.000304)\n",
      "392. feature 174 (0.000289)\n",
      "393. feature 664 (0.000285)\n",
      "394. feature 610 (0.000273)\n",
      "395. feature 536 (0.000270)\n",
      "396. feature 691 (0.000269)\n",
      "397. feature 716 (0.000264)\n",
      "398. feature 147 (0.000264)\n",
      "399. feature 621 (0.000263)\n",
      "400. feature 583 (0.000258)\n",
      "401. feature 313 (0.000254)\n",
      "402. feature 257 (0.000236)\n",
      "403. feature 94 (0.000235)\n",
      "404. feature 133 (0.000235)\n",
      "405. feature 742 (0.000231)\n",
      "406. feature 201 (0.000222)\n",
      "407. feature 360 (0.000205)\n",
      "408. feature 620 (0.000204)\n",
      "409. feature 528 (0.000201)\n",
      "410. feature 285 (0.000195)\n",
      "411. feature 472 (0.000194)\n",
      "412. feature 740 (0.000192)\n",
      "413. feature 739 (0.000190)\n",
      "414. feature 104 (0.000188)\n",
      "415. feature 388 (0.000187)\n",
      "416. feature 564 (0.000186)\n",
      "417. feature 556 (0.000184)\n",
      "418. feature 120 (0.000182)\n",
      "419. feature 592 (0.000180)\n",
      "420. feature 221 (0.000179)\n",
      "421. feature 416 (0.000179)\n",
      "422. feature 638 (0.000178)\n",
      "423. feature 134 (0.000173)\n",
      "424. feature 229 (0.000172)\n",
      "425. feature 444 (0.000170)\n",
      "426. feature 500 (0.000170)\n",
      "427. feature 105 (0.000168)\n",
      "428. feature 611 (0.000167)\n",
      "429. feature 744 (0.000166)\n",
      "430. feature 508 (0.000161)\n",
      "431. feature 249 (0.000158)\n",
      "432. feature 649 (0.000158)\n",
      "433. feature 665 (0.000155)\n",
      "434. feature 284 (0.000154)\n",
      "435. feature 741 (0.000150)\n",
      "436. feature 146 (0.000150)\n",
      "437. feature 692 (0.000147)\n",
      "438. feature 277 (0.000146)\n",
      "439. feature 743 (0.000145)\n",
      "440. feature 68 (0.000139)\n",
      "441. feature 706 (0.000137)\n",
      "442. feature 164 (0.000135)\n",
      "443. feature 332 (0.000135)\n",
      "444. feature 72 (0.000133)\n",
      "445. feature 93 (0.000130)\n",
      "446. feature 720 (0.000130)\n",
      "447. feature 92 (0.000129)\n",
      "448. feature 71 (0.000128)\n",
      "449. feature 452 (0.000126)\n",
      "450. feature 368 (0.000124)\n",
      "451. feature 480 (0.000122)\n",
      "452. feature 677 (0.000119)\n",
      "453. feature 584 (0.000117)\n",
      "454. feature 736 (0.000116)\n",
      "455. feature 256 (0.000115)\n",
      "456. feature 173 (0.000112)\n",
      "457. feature 69 (0.000111)\n",
      "458. feature 70 (0.000111)\n",
      "459. feature 424 (0.000111)\n",
      "460. feature 535 (0.000109)\n",
      "461. feature 648 (0.000108)\n",
      "462. feature 340 (0.000104)\n",
      "463. feature 135 (0.000103)\n",
      "464. feature 529 (0.000103)\n",
      "465. feature 693 (0.000102)\n",
      "466. feature 396 (0.000102)\n",
      "467. feature 119 (0.000099)\n",
      "468. feature 666 (0.000098)\n",
      "469. feature 745 (0.000098)\n",
      "470. feature 746 (0.000097)\n",
      "471. feature 738 (0.000094)\n",
      "472. feature 73 (0.000094)\n",
      "473. feature 193 (0.000092)\n",
      "474. feature 228 (0.000092)\n",
      "475. feature 563 (0.000092)\n",
      "476. feature 312 (0.000091)\n",
      "477. feature 747 (0.000088)\n",
      "478. feature 74 (0.000087)\n",
      "479. feature 737 (0.000082)\n",
      "480. feature 200 (0.000081)\n",
      "481. feature 639 (0.000080)\n",
      "482. feature 305 (0.000078)\n",
      "483. feature 75 (0.000077)\n",
      "484. feature 479 (0.000072)\n",
      "485. feature 501 (0.000072)\n",
      "486. feature 165 (0.000065)\n",
      "487. feature 145 (0.000064)\n",
      "488. feature 106 (0.000060)\n",
      "489. feature 676 (0.000059)\n",
      "490. feature 612 (0.000058)\n",
      "491. feature 473 (0.000058)\n",
      "492. feature 66 (0.000056)\n",
      "493. feature 694 (0.000056)\n",
      "494. feature 445 (0.000054)\n",
      "495. feature 136 (0.000054)\n",
      "496. feature 591 (0.000053)\n",
      "497. feature 667 (0.000053)\n",
      "498. feature 417 (0.000051)\n",
      "499. feature 640 (0.000049)\n",
      "500. feature 172 (0.000048)\n",
      "501. feature 507 (0.000048)\n",
      "502. feature 705 (0.000046)\n",
      "503. feature 67 (0.000046)\n",
      "504. feature 585 (0.000044)\n",
      "505. feature 222 (0.000039)\n",
      "506. feature 721 (0.000039)\n",
      "507. feature 227 (0.000038)\n",
      "508. feature 695 (0.000038)\n",
      "509. feature 250 (0.000038)\n",
      "510. feature 361 (0.000036)\n",
      "511. feature 283 (0.000036)\n",
      "512. feature 557 (0.000035)\n",
      "513. feature 722 (0.000035)\n",
      "514. feature 118 (0.000035)\n",
      "515. feature 339 (0.000035)\n",
      "516. feature 278 (0.000034)\n",
      "517. feature 76 (0.000034)\n",
      "518. feature 389 (0.000033)\n",
      "519. feature 451 (0.000030)\n",
      "520. feature 395 (0.000030)\n",
      "521. feature 108 (0.000030)\n",
      "522. feature 311 (0.000029)\n",
      "523. feature 306 (0.000028)\n",
      "524. feature 647 (0.000028)\n",
      "525. feature 333 (0.000028)\n",
      "526. feature 255 (0.000027)\n",
      "527. feature 367 (0.000027)\n",
      "528. feature 704 (0.000026)\n",
      "529. feature 668 (0.000025)\n",
      "530. feature 199 (0.000023)\n",
      "531. feature 423 (0.000023)\n",
      "532. feature 675 (0.000023)\n",
      "533. feature 474 (0.000022)\n",
      "534. feature 613 (0.000021)\n",
      "535. feature 65 (0.000021)\n",
      "536. feature 137 (0.000021)\n",
      "537. feature 418 (0.000020)\n",
      "538. feature 194 (0.000020)\n",
      "539. feature 77 (0.000019)\n",
      "540. feature 107 (0.000019)\n",
      "541. feature 619 (0.000018)\n",
      "542. feature 748 (0.000018)\n",
      "543. feature 558 (0.000017)\n",
      "544. feature 390 (0.000016)\n",
      "545. feature 116 (0.000015)\n",
      "546. feature 502 (0.000015)\n",
      "547. feature 734 (0.000015)\n",
      "548. feature 144 (0.000015)\n",
      "549. feature 117 (0.000015)\n",
      "550. feature 91 (0.000015)\n",
      "551. feature 534 (0.000014)\n",
      "552. feature 171 (0.000014)\n",
      "553. feature 446 (0.000014)\n",
      "554. feature 530 (0.000013)\n",
      "555. feature 749 (0.000012)\n",
      "556. feature 44 (0.000012)\n",
      "557. feature 641 (0.000012)\n",
      "558. feature 735 (0.000012)\n",
      "559. feature 64 (0.000011)\n",
      "560. feature 362 (0.000011)\n",
      "561. feature 90 (0.000010)\n",
      "562. feature 254 (0.000010)\n",
      "563. feature 506 (0.000010)\n",
      "564. feature 39 (0.000010)\n",
      "565. feature 334 (0.000010)\n",
      "566. feature 279 (0.000010)\n",
      "567. feature 590 (0.000010)\n",
      "568. feature 503 (0.000010)\n",
      "569. feature 750 (0.000010)\n",
      "570. feature 63 (0.000009)\n",
      "571. feature 776 (0.000009)\n",
      "572. feature 618 (0.000008)\n",
      "573. feature 49 (0.000008)\n",
      "574. feature 45 (0.000008)\n",
      "575. feature 586 (0.000008)\n",
      "576. feature 732 (0.000008)\n",
      "577. feature 198 (0.000008)\n",
      "578. feature 46 (0.000008)\n",
      "579. feature 723 (0.000007)\n",
      "580. feature 43 (0.000007)\n",
      "581. feature 562 (0.000007)\n",
      "582. feature 78 (0.000007)\n",
      "583. feature 475 (0.000007)\n",
      "584. feature 669 (0.000007)\n",
      "585. feature 771 (0.000007)\n",
      "586. feature 166 (0.000007)\n",
      "587. feature 79 (0.000006)\n",
      "588. feature 478 (0.000006)\n",
      "589. feature 41 (0.000006)\n",
      "590. feature 366 (0.000006)\n",
      "591. feature 773 (0.000006)\n",
      "592. feature 769 (0.000006)\n",
      "593. feature 770 (0.000005)\n",
      "594. feature 310 (0.000005)\n",
      "595. feature 36 (0.000005)\n",
      "596. feature 40 (0.000005)\n",
      "597. feature 775 (0.000005)\n",
      "598. feature 772 (0.000004)\n",
      "599. feature 724 (0.000004)\n",
      "600. feature 251 (0.000004)\n",
      "601. feature 335 (0.000004)\n",
      "602. feature 282 (0.000004)\n",
      "603. feature 733 (0.000004)\n",
      "604. feature 363 (0.000004)\n",
      "605. feature 765 (0.000004)\n",
      "606. feature 751 (0.000004)\n",
      "607. feature 768 (0.000004)\n",
      "608. feature 338 (0.000004)\n",
      "609. feature 764 (0.000004)\n",
      "610. feature 614 (0.000004)\n",
      "611. feature 42 (0.000004)\n",
      "612. feature 12 (0.000004)\n",
      "613. feature 226 (0.000004)\n",
      "614. feature 777 (0.000004)\n",
      "615. feature 763 (0.000004)\n",
      "616. feature 766 (0.000004)\n",
      "617. feature 394 (0.000004)\n",
      "618. feature 143 (0.000003)\n",
      "619. feature 223 (0.000003)\n",
      "620. feature 767 (0.000003)\n",
      "621. feature 674 (0.000003)\n",
      "622. feature 450 (0.000002)\n",
      "623. feature 47 (0.000002)\n",
      "624. feature 703 (0.000002)\n",
      "625. feature 696 (0.000002)\n",
      "626. feature 225 (0.000002)\n",
      "627. feature 307 (0.000002)\n",
      "628. feature 38 (0.000002)\n",
      "629. feature 89 (0.000002)\n",
      "630. feature 393 (0.000002)\n",
      "631. feature 646 (0.000002)\n",
      "632. feature 87 (0.000002)\n",
      "633. feature 774 (0.000002)\n",
      "634. feature 642 (0.000002)\n",
      "635. feature 559 (0.000002)\n",
      "636. feature 420 (0.000002)\n",
      "637. feature 617 (0.000002)\n",
      "638. feature 35 (0.000002)\n",
      "639. feature 109 (0.000002)\n",
      "640. feature 391 (0.000002)\n",
      "641. feature 762 (0.000002)\n",
      "642. feature 422 (0.000001)\n",
      "643. feature 170 (0.000001)\n",
      "644. feature 195 (0.000001)\n",
      "645. feature 2 (0.000000)\n",
      "646. feature 3 (0.000000)\n",
      "647. feature 80 (0.000000)\n",
      "648. feature 81 (0.000000)\n",
      "649. feature 82 (0.000000)\n",
      "650. feature 22 (0.000000)\n",
      "651. feature 21 (0.000000)\n",
      "652. feature 20 (0.000000)\n",
      "653. feature 19 (0.000000)\n",
      "654. feature 18 (0.000000)\n",
      "655. feature 17 (0.000000)\n",
      "656. feature 16 (0.000000)\n",
      "657. feature 15 (0.000000)\n",
      "658. feature 14 (0.000000)\n",
      "659. feature 13 (0.000000)\n",
      "660. feature 23 (0.000000)\n",
      "661. feature 10 (0.000000)\n",
      "662. feature 1 (0.000000)\n",
      "663. feature 9 (0.000000)\n",
      "664. feature 8 (0.000000)\n",
      "665. feature 7 (0.000000)\n",
      "666. feature 6 (0.000000)\n",
      "667. feature 5 (0.000000)\n",
      "668. feature 83 (0.000000)\n",
      "669. feature 4 (0.000000)\n",
      "670. feature 11 (0.000000)\n",
      "671. feature 24 (0.000000)\n",
      "672. feature 25 (0.000000)\n",
      "673. feature 85 (0.000000)\n",
      "674. feature 62 (0.000000)\n",
      "675. feature 61 (0.000000)\n",
      "676. feature 60 (0.000000)\n",
      "677. feature 59 (0.000000)\n",
      "678. feature 58 (0.000000)\n",
      "679. feature 57 (0.000000)\n",
      "680. feature 56 (0.000000)\n",
      "681. feature 55 (0.000000)\n",
      "682. feature 54 (0.000000)\n",
      "683. feature 53 (0.000000)\n",
      "684. feature 52 (0.000000)\n",
      "685. feature 51 (0.000000)\n",
      "686. feature 26 (0.000000)\n",
      "687. feature 50 (0.000000)\n",
      "688. feature 48 (0.000000)\n",
      "689. feature 37 (0.000000)\n",
      "690. feature 34 (0.000000)\n",
      "691. feature 33 (0.000000)\n",
      "692. feature 32 (0.000000)\n",
      "693. feature 31 (0.000000)\n",
      "694. feature 30 (0.000000)\n",
      "695. feature 29 (0.000000)\n",
      "696. feature 28 (0.000000)\n",
      "697. feature 27 (0.000000)\n",
      "698. feature 84 (0.000000)\n",
      "699. feature 783 (0.000000)\n",
      "700. feature 86 (0.000000)\n",
      "701. feature 671 (0.000000)\n",
      "702. feature 702 (0.000000)\n",
      "703. feature 701 (0.000000)\n",
      "704. feature 700 (0.000000)\n",
      "705. feature 699 (0.000000)\n",
      "706. feature 698 (0.000000)\n",
      "707. feature 697 (0.000000)\n",
      "708. feature 673 (0.000000)\n",
      "709. feature 672 (0.000000)\n",
      "710. feature 670 (0.000000)\n",
      "711. feature 560 (0.000000)\n",
      "712. feature 645 (0.000000)\n",
      "713. feature 644 (0.000000)\n",
      "714. feature 643 (0.000000)\n",
      "715. feature 616 (0.000000)\n",
      "716. feature 615 (0.000000)\n",
      "717. feature 589 (0.000000)\n",
      "718. feature 588 (0.000000)\n",
      "719. feature 587 (0.000000)\n",
      "720. feature 725 (0.000000)\n",
      "721. feature 726 (0.000000)\n",
      "722. feature 727 (0.000000)\n",
      "723. feature 728 (0.000000)\n",
      "724. feature 781 (0.000000)\n",
      "725. feature 780 (0.000000)\n",
      "726. feature 779 (0.000000)\n",
      "727. feature 778 (0.000000)\n",
      "728. feature 761 (0.000000)\n",
      "729. feature 760 (0.000000)\n",
      "730. feature 759 (0.000000)\n",
      "731. feature 758 (0.000000)\n",
      "732. feature 757 (0.000000)\n",
      "733. feature 756 (0.000000)\n",
      "734. feature 755 (0.000000)\n",
      "735. feature 754 (0.000000)\n",
      "736. feature 753 (0.000000)\n",
      "737. feature 752 (0.000000)\n",
      "738. feature 731 (0.000000)\n",
      "739. feature 730 (0.000000)\n",
      "740. feature 729 (0.000000)\n",
      "741. feature 561 (0.000000)\n",
      "742. feature 533 (0.000000)\n",
      "743. feature 88 (0.000000)\n",
      "744. feature 141 (0.000000)\n",
      "745. feature 252 (0.000000)\n",
      "746. feature 224 (0.000000)\n",
      "747. feature 197 (0.000000)\n",
      "748. feature 196 (0.000000)\n",
      "749. feature 169 (0.000000)\n",
      "750. feature 168 (0.000000)\n",
      "751. feature 167 (0.000000)\n",
      "752. feature 142 (0.000000)\n",
      "753. feature 140 (0.000000)\n",
      "754. feature 532 (0.000000)\n",
      "755. feature 139 (0.000000)\n",
      "756. feature 138 (0.000000)\n",
      "757. feature 115 (0.000000)\n",
      "758. feature 114 (0.000000)\n",
      "759. feature 113 (0.000000)\n",
      "760. feature 112 (0.000000)\n",
      "761. feature 111 (0.000000)\n",
      "762. feature 110 (0.000000)\n",
      "763. feature 253 (0.000000)\n",
      "764. feature 280 (0.000000)\n",
      "765. feature 281 (0.000000)\n",
      "766. feature 308 (0.000000)\n",
      "767. feature 531 (0.000000)\n",
      "768. feature 505 (0.000000)\n",
      "769. feature 504 (0.000000)\n",
      "770. feature 477 (0.000000)\n",
      "771. feature 476 (0.000000)\n",
      "772. feature 449 (0.000000)\n",
      "773. feature 448 (0.000000)\n",
      "774. feature 447 (0.000000)\n",
      "775. feature 421 (0.000000)\n",
      "776. feature 419 (0.000000)\n",
      "777. feature 392 (0.000000)\n",
      "778. feature 782 (0.000000)\n",
      "779. feature 365 (0.000000)\n",
      "780. feature 364 (0.000000)\n",
      "781. feature 337 (0.000000)\n",
      "782. feature 336 (0.000000)\n",
      "783. feature 309 (0.000000)\n",
      "784. feature 0 (0.000000)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEICAYAAABMGMOEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+QZWV95/H3J92AhsZuHdQAwzoYJlZQk45OkJSTTK8oAlGGJFCOMYFskYxxQ2UtN2sguxiXjVWSSjRriT9QVGIWgZBVZ3UMGrFnq40iM3oVBiQ2iDvNIGCG6UwTEefmu3+c5/ScvnN/nO57u++vz6vq1pz7nOec8713Zs73Ps9znnMUEZiZmbXyE90OwMzM+oMThpmZleKEYWZmpThhmJlZKU4YZmZWihOGmZmV4oRhtkKSPiDpqm7HYbZW5HkYttYkPQg8F6gWin8mIva3sc8p4G8iYn170fUnSR8D5iLiv3U7FhtcbmFYt7w2IsYKrxUni06QNNrN47dD0ki3Y7Dh4IRhPUXSWZL+UdJBSd9MLYd83X+QdK+kQ5IekPTGVH488DngZEkL6XWypI9J+rPC9lOS5grvH5T0x5K+BTwhaTRt93eSHpP0XUl/2CTWxf3n+5b0VkmPSnpY0oWSzpf0T5IOSPqTwrZvl3SrpJvT5/m6pJ8vrP9ZSdPpe9gr6YKa475f0k5JTwCXAW8A3po++/9J9a6QdH/a/z2Sfq2wj9+RNCPpLyQ9nj7reYX1z5L0UUn70/pPFda9RlIlxfaPkn6usO6PJT2UjnmfpLNL/LVbv4gIv/xa0xfwIPDKOuWnAP8MnE/2Y+ZV6f2z0/pfBX4aELAF+FfgJWndFFmXTHF/HwP+rPB+SZ0URwU4FXh6OuYe4G3AscDzgQeAVzf4HIv7T/s+nLY9Bvg94DHgRuAE4IXAk8DzU/23Az8GLkr1/wj4blo+BpgF/iTF8QrgEPCCwnHngZenmJ9W+1lTvYuBk1Od1wFPACeldb+Tjv97wAjwJmA/R7qpPwvcDDwzxbMllb8EeBR4Wdru0vQ9Hge8ANgHnJzqbgB+utv/3vzq3MstDOuWT6VfqAcLv15/C9gZETsj4t8i4gvAbrIEQkR8NiLuj8wu4PPAL7cZx3siYl9E/BD4RbLkdHVEPBURDwAfAraV3NePgXdExI+Bm4ATgf8ZEYciYi+wF/i5Qv09EXFrqv8ushP/Wek1BrwzxXE78Bng9YVtPx0RX07f05P1gomIv42I/anOzcB3gDMLVb4XER+KiCpwA3AS8FxJJwHnAb8fEY9HxI/T9w1ZgvlgRNwREdWIuAH4UYq5SpY4zpB0TEQ8GBH3l/zurA84YVi3XBgRE+l1YSp7HnBxIZEcBDaTnciQdJ6kr6bunYNkieTENuPYV1h+Hlm3VvH4f0I2QF/GP6eTL8AP05+PFNb/kCwRHHXsiPg3YI6sRXAysC+V5b5H1gKrF3ddki4pdB0dBF7E0u/r+4Xj/2taHCNrcR2IiMfr7PZ5wH+u+Y5OJWtVzAJvJms9PSrpJkknt4rT+ocThvWSfcDHC4lkIiKOj4h3SjoO+DvgL4DnRsQEsJOsewqg3uV+TwA/WXj/U3XqFLfbB3y35vgnRMT5bX+y+k7NFyT9BLCerFtoP3BqKsv9O+ChBnEf9V7S88haR5cD69L3dTdHvq9m9gHPkjTRYN07ar6jn4yITwBExI0RsZkssQRwTYnjWZ9wwrBe8jfAayW9WtKIpKelweT1ZH35x5GNCxxOA7TnFLZ9BFgnabxQVgHOTwO4P0X267eZrwH/kgZun55ieJGkX+zYJ1zqpZJ+XdkVWm8m69r5KnAHWbJ7q6Rj0sD/a8m6uRp5hGzMJXc82Qn7McguGCBrYbQUEQ+TXUTwPknPTDH8Slr9IeD3Jb1MmeMl/aqkEyS9QNIrUnJ/kqxFVW1wGOtDThjWMyJiH7CVrBvoMbJfs/8F+ImIOAT8IXAL8Djwm8COwrbfBj4BPJC6Sk4GPg58k2xQ9vNkg7jNjl8lOzFPkg1A/wD4MDDebLs2fJpsMPpx4LeBX0/jBU8BF5CNI/wAeB9wSfqMjVxPNnZwUNKnIuIe4C+Br5AlkxcDX15GbL9NNibzbbJB7jcDRMRusnGM96a4Z8kG0CFL6O9MMX8feA7Z36UNCE/cM+sCSW8HTo+I3+p2LGZluYVhZmalOGGYmVkp7pIyM7NS3MIwM7NS+vaGa/WceOKJsWHDhm6HYWbWV/bs2fODiHh2q3oDlTA2bNjA7t27ux2GmVlfkfS9MvXcJWVmZqU4YZiZWSlOGGZmVkpHEoakc9PDUmYlXVFn/XHpQTGzku6QtCGVv0rSHkl3pT9fUdhmOu2zkl7P6USsZma2Mm0Peit7POS1ZA+7mQPulLQj3csmdxnweEScLmkb2R0sX0d2z5nXRsR+SS8CbmPpLZzfkO5dY2ZmXdaJFsaZwGxEPJBumnYT2Q3kiraSPaAF4FbgbEmKiG/EkWc57wWelu50aWZmPaYTCeMUlj7MZY6lrYQldSLiMNnjJdfV1PkN4BsR8aNC2UdTd9RVksrcx9/MzFZJJxJGvRN57f1GmtaR9EKybqo3Fta/ISJeTPYIzl8mu93y0QeXtkvaLWn3Y489tqzAzcysvE4kjDkKTw7jyFPD6tZJD4sZBw6k9+uBT5Ld73/x+b8R8VD68xBwI0ufRUyh3nURsSkiNh04cKADH8fMzOrpRMK4E9go6TRJxwLbKDzYJtkBXJqWLwJuj4hIj4D8LHBlRCw+3EXSqKQT0/IxwGvIHi9pZmZd0nbCSGMSl5Nd4XQvcEtE7JV0taQLUrXryR6fOQu8Bcgvvb0cOB24quby2eOA2yR9i+wxmw+RPRrSzMy6ZKBub37CCSfEoUOHuh2GmVlfkbQnIja1queZ3mZmVooThpmZleKEYWZmpThhmJlZKU4YZmZWihOGmZmV4oRhZmalOGGYmVkpThhmZlaKE4aZmZUyUAnjhz/8IVNTU90Ow8xsIA1UwjAzs9XjhGFmZqU4YZiZWSlOGGZmVooThpmZleKEYWZmpThhmJlZKU4YZmZWysAljEql4sl7ZmarYOAShpmZrQ4nDDMzK8UJw8zMSnHCMDOzUpwwzMyslI4kDEnnSrpP0qykK+qsP07SzWn9HZI2pPJXSdoj6a705ysK27w0lc9Keo8kdSJWMzNbmbYThqQR4FrgPOAM4PWSzqipdhnweEScDrwbuCaV/wB4bUS8GLgU+Hhhm/cD24GN6XVuu7GamdnKdaKFcSYwGxEPRMRTwE3A1po6W4Eb0vKtwNmSFBHfiIj9qXwv8LTUGjkJeEZEfCUiAvhr4MJWgVSrVRYWFjrwkczMrFYnEsYpwL7C+7lUVrdORBwG5oF1NXV+A/hGRPwo1Z9rsU8AJG2XtFvS7hV/AjMza2m0A/uoN7YQy6kj6YVk3VTnLGOfWWHEdcB1aT9165iZWfs60cKYA04tvF8P7G9UR9IoMA4cSO/XA58ELomI+wv117fYp5mZraFOJIw7gY2STpN0LLAN2FFTZwfZoDbARcDtERGSJoDPAldGxJfzyhHxMHBI0lnp6qhLgE93IFYzM1uhthNGGpO4HLgNuBe4JSL2Srpa0gWp2vXAOkmzwFuA/NLby4HTgaskVdLrOWndm4APA7PA/cDnysRTrVapVCrtfiwzM6uh7CKkwZCPYYyPj3Pw4MFuh2Nm1hck7YmITa3qeaa3mZmV4oRhZmalOGGYmVkpThhmZlaKE4aZmZXihGFmZqU4YZiZWSlOGGZmVooThpmZleKEYWZmpThhmJlZKU4YZmZWihOGmZmV4oRhZmalDHzCmJqaYmpqqtthmJn1vYFPGGZm1hkDmTAWFhbcqjAz67CBTBhmZtZ5ThhmZlaKE4aZmZXihGFmZqU4YZiZWSlOGGZmVspotwNYbZVKpdshmJkNhI60MCSdK+k+SbOSrqiz/jhJN6f1d0jakMrXSfqSpAVJ763ZZjrts5Jez+lErGZmtjJttzAkjQDXAq8C5oA7Je2IiHsK1S4DHo+I0yVtA64BXgc8CVwFvCi9ar0hIna3G6OZmbWvEy2MM4HZiHggIp4CbgK21tTZCtyQlm8FzpakiHgiImbIEkdHVSoVJiYmWFhY8MxvM7MO6ETCOAXYV3g/l8rq1omIw8A8sK7Evj+auqOukqR6FSRtl7RbklsiZmarqBMJo96JPFZQp9YbIuLFwC+n12/XqxQR10XEpojY1DLSxHewNTNbvk4kjDng1ML79cD+RnUkjQLjwIFmO42Ih9Kfh4Abybq+zMysSzqRMO4ENko6TdKxwDZgR02dHcClafki4PaIaNjCkDQq6cS0fAzwGuDu5QSVj13kKpWKWxVmZm1o+yqpiDgs6XLgNmAE+EhE7JV0NbA7InYA1wMflzRL1rLYlm8v6UHgGcCxki4EzgG+B9yWksUI8A/Ah5YbW7VaBWBkZKRl3TyZTE9PL/cwZmZDoSMT9yJiJ7CzpuxtheUngYsbbLuhwW5f2onYzMysM3xrEDMzK8UJw8zMShnIe0nlYxdmZtY5A5kwlsNXTpmZlTOUCaPZHWx9tZSZWX1DNYbhuRhmZis3lC2M4oQ+ONLimJyc7EY4ZmZ9YahaGGZmtnJD08KobVWYmdnyDE3C8KW2ZmbtcZdUQaVS8TPAzcwacMIwM7NSnDBqLCwslG5l+EFMZjZMnDA4+tkZjZ4B7gRhZsPMCaOOarXasJXhyX9mNqyGImEUr5Dq5qC2Wyhm1s+GImGYmVn7hmYeRm5hYWGxxTExMdFwnZmZLeUWRklTU1Oeo2FmQ22oE0bZ24U4WZiZDXnCaKbRpbUr5aRjZv1u6BKGxyjMzFZm6BLGSlQqFd/t1syG3lAnjGq1yvz8vFsdZmYlDN1lte1wK8PMhllHWhiSzpV0n6RZSVfUWX+cpJvT+jskbUjl6yR9SdKCpPfWbPNSSXelbd4jSZ2ItZ94ZriZ9ZK2E4akEeBa4DzgDOD1ks6oqXYZ8HhEnA68G7gmlT8JXAX8UZ1dvx/YDmxMr3PbjXW5KpUKExMTblmYmdGZFsaZwGxEPBARTwE3AVtr6mwFbkjLtwJnS1JEPBERM2SJY5Gkk4BnRMRXIiKAvwYu7ECsPcktCTPrB51IGKcA+wrv51JZ3ToRcRiYB9a12Odci30CIGm7pN2Sdi8z7r7k5GJm3dKJQe96Ywuxgjorqh8R1wHXAUhqtk8zM2tDJ1oYc8Cphffrgf2N6kgaBcaBAy32ub7FPtdUtVr15bdmNtQ6kTDuBDZKOk3SscA2YEdNnR3ApWn5IuD2NDZRV0Q8DBySdFa6OuoS4NMdiLWhsgmhWq0yMzPTkW6hZt1LvpWImfWathNGGpO4HLgNuBe4JSL2Srpa0gWp2vXAOkmzwFuAxUtvJT0IvAv4HUlzhSus3gR8GJgF7gc+126sndLsiXxmZoOqIxP3ImInsLOm7G2F5SeBixtsu6FB+W7gRZ2Iz8zM2jfUtwYpYyUPVer2lUzdPr6ZDSbfGqQLfDI3s37khNFAfmPCZvJxjDwBTE9PH1XHycHMBoW7pMzMrBQnDDMzK8UJY4UWFhaW3JSwUqm4+8nMBpoThpmZleKEYWZmpfgqqQHUaBZ6s6u5zMxacQujAyqVSsuHLJWpk/N9pMysF7mF0YZqtcrCwgJjY2NLyosn+9U68bu1YGZrzS0MMzMrxQmjg2ovrZ2YmFicLV72nlTujjKzXuWEsUL5yT/vlupnrZ7L4fklZgZOGB2RJ4+FhQVmZmaaJpBKpVK6BbGwsFC37nL2YWbWKU4YZmZWihNGh3Timd9rNX7hbiYzWwknjFVSvNfUSh7CVGb/tcllamqKiYmJJV1iZZOD74VlZq04YXRYsaXRLEnMzMwwMzOzpKze5L6FhQWfyM2sJzhhWF3FlolbH2YGnum9ahq1LvIWRLVaZWRkpOH2eTdWsU6zMQ7P3zCz1eaEsQaadU0t5x5TZmbd5ISxxmqTR6sB8Wq1yszMTOkuobUaYDez4eMxjD7W6QFxj1WYWTMdSRiSzpV0n6RZSVfUWX+cpJvT+jskbSisuzKV3yfp1YXyByXdJakiaXcn4uw1jeZu1HZRVavVxdnd8/Pzbd8mvXamuLvFzKyMtrukJI0A1wKvAuaAOyXtiIh7CtUuAx6PiNMlbQOuAV4n6QxgG/BC4GTgHyT9TETkZ9F/HxE/aDfGftMoidTeRj2XJ4Zid5S7kcys0zrRwjgTmI2IByLiKeAmYGtNna3ADWn5VuBsSUrlN0XEjyLiu8Bs2p/V6MebHHpGudlg6cSg9ynAvsL7OeBljepExGFJ88C6VP7Vmm1PScsBfF5SAB+MiOvqHVzSdmB7ux+iH+UD4s24pWFmndKJhKE6ZVGyTrNtXx4R+yU9B/iCpG9HxP89qnKWSK4DSMll4BXnZ7Saz1FPv7VUzKw3dKJLag44tfB+PbC/UR1Jo8A4cKDZthGR//ko8EncVdXxy2Ub8SRAM6unEwnjTmCjpNMkHUs2iL2jps4O4NK0fBFwe0REKt+WrqI6DdgIfE3S8ZJOAJB0PHAOcHcHYh0YZZLHaszJMLPh1XaXVBqTuBy4DRgBPhIReyVdDeyOiB3A9cDHJc2StSy2pW33SroFuAc4DPxBRFQlPRf4ZDYuzihwY0T8fbuxDionBTNbCx2Z6R0RO4GdNWVvKyw/CVzcYNt3AO+oKXsA+PlOxDasapPI1NQUMzMzSy7NLV6Oa2bWimd627LUu1TWl8+aDQcnjCG33HGO2lni7Q6QO9mY9Q8njCFQnIuxkoHw2kmDfqiT2XDy3WqHRL3na8DRYx2rNaZRTDCVSoXJyUmmp6fb3l87+zCz5XHCGALFpFB8fGztLPFi6yNPLI26m7p1Z1u3bMy6x11SQ6zVczjqJQvP7TAbXk4Y1lCzrqn5+fmjbpFe/PVfHAzPL+ld7uC4B8TNeou7pGxRbcuhONhdrVaPOnnng+nLGfPwvA+z/uWEYcDKZovX26Y2qZRpVeStEw9gm/U2JwxrqlUiKV59lSeHycnJjhy72KUF9a+I8tVSZmvHYxhW2szMTMMEUu9JfwsLC4uPlO3HB0CZ2VJuYVhprVob8/Pzi8uVSmXJJbz5n3nymJmZWRwMN7P+4BaG9YRG8zoWFhZaXmFV3NZXVpmtHicMa0u9VkfeDdVqu127dlGtVhdbHpVKhYmJCSYmJuqe9Jdz3yonDrPOU/Yco8EwLI9oHUQjIyOLt14vJpvizPOxsTEWFhaW3KJ9cnKy6WC7B8PNWpO0JyI2tarnMQzrGfkYyMjISMM5IdVqdbHe+Pj4kjqdvkrLzJZyl5T1hHr3u2pWB5bONl9YWFh8Neq2mpqaatrlZWbNuYVhfa14ZVYuHygfGxtbHPfIu65qu7Tg6Lkcrd6bDSsnjBWq120C2UPNc9VCWb3fzI32Ye3Jv9P5+fnFy3brtTqKg+iTk5NudZi14ITRAY0SQt26IyNQrbZMJo22dYJZnnwWenF8JFepVBbL81ZIkROI2VK+SmqFiifvkSb1qixNCuPj4yzMz2flIyOMVassUC5p1B6zbNIpJqlOJ53lJLxekieO4lVY+XLtYHquzEOf3H1l/ajsVVJDN+jd6OQ+MjLS9MTfbH9jdcrHGpTXmkz1ir98x8fHm8ZSb91Ig3KAsbExNtdZXzxm7ZP46pXXP402iLHB/npFPv+j+D6XzyPJX/Pz84stEc/vsGE2FAljtU5djZJFrbGaGGrfTwKbN29mnMaxjsNRg7WtYmuVdDZv3nxUWTOTJepA81jLJKZeSDZ5QikmkpmZmcVZ56Ojo4yOjjZMHvkVWU4uNkgGMmG0OlmW3cc4yzt5jQGbKZdEOmW5n3NsbKz0NiMsL0k1S3hlt19OK2atFWel58szMzNLEkOlUllyqe9yHxpl1ssGMmGshbGxsbYSw2aO/oW/XHkLp0z313SDOs1O8MX6I2kfzWIpmpycbNnCyccK6iWm2qTfzo+ATrdYaruy5ufn2bVrF5IWu6+a3dkXPCfE+lNHEoakcyXdJ2lW0hV11h8n6ea0/g5JGwrrrkzl90l6ddl99qJJmp9UV2JkZITxtO/Jycmsu6fmmI3mNdd2fbUyxpFW1UiT/S7GVucYY4Xt8nX5CbuYGMZYmlQmJyc5mLbNx1wWt+dI6602ibRq1eTr8tZL/n0ut/W4XMVLeyUd9dq1a9eSiYYTExOLXVzFcZJ6XVvFRFObbDzGYqup7YQhaQS4FjgPOAN4vaQzaqpdBjweEacD7wauSdueAWwDXgicC7xP0kjJfa7Ikl/NJQa6l9OFU6tsn3+xfh7fck/2Zfdfe+loo/GUPI5Gt9nIT+LFtfWS1xhHvsOxwraTZFcS1XbhTQMHDx5smHjHgC2wmDjz72ycIxcL5MvjpC7CsbGj4soTU654ocFIzav4makpz/8NjQBbtmxZUt5K3q2Vt0ryGzLmrzyxFFswxVZMsZ4kJiYmmJmZYdeuXYyOjjZswXh8xVaqEy2MM4HZiHggIp4CbgK21tTZCtyQlm8FzpakVH5TRPwoIr4LzKb9ldnnitSexDtxYi6e6Ium65RPTk62/OVeb//TdcrHGpSXNT09vexY8u+rtiWRf87pmpjqJanlyo81XThmo5ZVXjf/nqebHD9vsY2n5c1kyWhz4VXbysmT0GEggMOHD3M4vc8T4BaypJsnr4ggUv18eQs1CS4dY0vNa6RQ3iwZ5Z+BYjfY/DwLqXustnVTm4TW4jUxMQFwVAvKiat/dGLi3inAvsL7OeBljepExGFJ88C6VP7Vmm1PScut9gmApO3A9iWF9e58mv9H2rIFZmaOvB8fh7xOmq+wuJz/Ai2WF+XX609OQqWS/Tk9zWT+H2B6GtJ/krFiLFu2HNlH7XI+SFqMa8uWbF9TU0eOubCQxZd+jeezlvMB2Tz+vOtncfv88yTTEUxMTGSxjY9nnyE/fvFEOzIChw9nnyc/dr4+TXpbMvcgjzWvU/t95uM3xc9cnBMUARMTR+LPv896J5fC8ZecfNL3lke15PbohXgXtyn+XST5N1Cp9xmLsRaPmYyNjx+JveazTRfiKTO/o556cz76YR5IMbZejtOO1omEoTpltRPoGtVpVF6v5VN3Ul5EXAdcB2t/e/Myv5zzOmWulslPeMW6Y2Nji/+p6q1vZGxsbMmtv9tRvPV4ru4JuqC4bnp6evHXZTcdlSBqynP17iW13JN6s++m0XGXq972PgHbaupEwpgDTi28Xw/sb1BnTtIoWQv6QIttW+1zWWqb8SMjI2zevHmgL3ssc9JaiWISK6uYXGu/80Yn43ybVsdaycm8nfWdOo5Zv+lEwrgT2CjpNOAhskHs36ypswO4FPgKcBFwe0SEpB3AjZLeBZwMbAS+RtbyaLXPuvJfw/nJplPPjC4mnOL+c61OWp36td9K/pChVnVWqtefNbEaJ+lOJiOzftb2oHdEHAYuB24D7gVuiYi9kq6WdEGqdj2wTtIs8BbgirTtXuAW4B7g74E/iIhqo322E+fY2FhbJ8qilfY5m5n1s47crTYidgI7a8reVlh+Eri4wbbvAN5RZp+dlnfZtNsK6WTyKI5T1GvJ5GrXLWd8o51WQu1nLdvt1Wj8YDnHWmkdM+uMgZ7pPT093Xb3y8GDBzvWMukX09PTq9b1ND09nc2z8InerO8M1PMw6l3NU5R3JY2ODtTH7jqf/M2Gw0C1MCYnJ3t+ULYTGn3O1WwZmJn5p3ZBfoVRuw8Yqjcxabl992XHI+ptu9w5D2t1BVc9y22drOT7NLPOGMqEUdtt1WqMopsnVDOzXjGQCaMTl72uZHJaL1rOZ+iny4X7JU6zQTKQCaOe4slwEPr5fcI0s7U2NAmjl3S7H97JxsxWYqCukmqk3e6lfO7AILRMVqLZBEIzGx5uYdTIT47+Fd4+f4dmg8UJo2CYf0XXJslud5uZWe8Z+ISxnGdWrLXVOil3+vO4xWVmMCRjGGZm1r6Bb2FYa249mFkZQ9/CmJ6eLn3C9L2azGyYuYWxTGv1a3w1Hq/qloSZtcMJowf08om8l2Mzs7U1lAmj2UnQJ0gzs/oGPmE4AZiZdcbAJ4zV5GRkZsPECSNZ7ZP/agxiNzuWmVmnOWH0GScDM+sWJ4we5uRgZr1k6CfumZlZOW0lDEnPkvQFSd9Jfz6zQb1LU53vSLq0UP5SSXdJmpX0HklK5W+X9JCkSnqd306cZmbWvnZbGFcAX4yIjcAX0/slJD0L+FPgZcCZwJ8WEsv7ge3AxvQ6t7DpuyNiMr12lg1oObf6MDOz8todw9gKTKXlG4Bp4I9r6rwa+EJEHACQ9AXgXEnTwDMi4iup/K+BC4HPtRlTz3IiM7N+1m4L47kR8TBA+vM5deqcAuwrvJ9LZaek5dry3OWSviXpI426ugAkbZe0W9Luxx57bKWfw8zMWmiZMCT9g6S767y2ljyG6pRFk3LIuqp+GpgEHgb+stHOI+K6iNgUEZue/exnlwzJzMyWq2WXVES8stE6SY9IOikiHpZ0EvBonWpzHOm2AlhP1nU1l5aL5fvTMR8pHONDwGdaxWlmZqur3S6pHUB+1dOlwKfr1LkNOEfSM1PX0jnAbakL65Cks9LVUZfk26fkk/s14O424zQzsza1O+j9TuAWSZcB/w+4GEDSJuD3I+J3I+KApP8B3Jm2uTofAAfeBHwMeDrZYHc+4P3nkibJuqgeBN7YZpxmZtYmRUTrWn1i06ZNsXv37m6HYWbWVyTtiYhNrep5preZmZXihGFmZqU4YZiZWSkDNYYh6RBwX7fjKOlE4AfdDqKkfom1X+IEx7oa+iVO6L1YnxcRLSeyDdrtze8rM3DTCyTtdqyd1S9xgmNdDf0SJ/RXrEXukjIzs1KcMMzMrJRBSxjXdTuAZXCsndcvcYJjXQ39Eif0V6yLBmrQ28zMVs+gtTDMzGyVOGGYmVkpA5MwJJ0r6b70fPCjHhXbhXg+IulRSXcXyuo+A12Z96TYvyXpJWsY56mSviTpXkl7Jf2nHo71aZK+JumbKdb/nspPk3RHivVmScem8uPS+9lXauELAAADxUlEQVS0fsNaxZqOPyLpG5I+0+NxPijpLkkVSbtTWc/9/afjT0i6VdK307/ZX+q1WCW9IH2X+etfJL251+JckYjo+xcwAtwPPB84FvgmcEaXY/oV4CXA3YWyPweuSMtXANek5fPJ7tQr4CzgjjWM8yTgJWn5BOCfgDN6NFYBY2n5GOCOFMMtwLZU/gHgTWn5PwIfSMvbgJvX+N/AW4Abgc+k970a54PAiTVlPff3n45/A/C7aflYYKJXY00xjADfB57Xy3GW/jzdDqBDfym/RPaMjfz9lcCVPRDXhpqEcR9wUlo+iWyiIcAHgdfXq9eFmD8NvKrXYwV+Evg68DKyGbOjtf8WyJ7F8ktpeTTV0xrFtx74IvAKsgeAqRfjTMeslzB67u8feAbw3drvphdjLRzzHODLvR5n2degdEk1em54r2n0DPSeiD91hfwC2S/3now1dfNUyJ7u+AWyluXBiDhcJ57FWNP6eWDdGoX6V8BbgX9L79f1aJyQPXfm85L2SNqeynrx7//5wGPAR1NX34clHd+jsea2AZ9Iy70cZymDkjCaPR+8H3Q9fkljwN8Bb46If2lWtU7ZmsUaEdWImCT7BX8m8LNN4ulKrJJeAzwaEXuKxU1i6fbf/8sj4iXAecAfSPqVJnW7GesoWTfv+yPiF4AnyLp2Gunq95rGqC4A/rZV1TplPXn+GpSEMQecWni/+HzwHvOI0uNntfQZ6F2NX9IxZMnif0XE/+7lWHMRcZDs2fBnAROS8vuiFeNZjDWtHwcOsPpeDlwg6UHgJrJuqb/qwTgBiIj96c9HgU+SJeJe/PufA+Yi4o70/layBNKLsUKWgL8eEY+k970aZ2mDkjDuBDamq1COJWsG7uhyTPU0egb6DuCSdLXEWcB83nRdbZIEXA/cGxHv6vFYny1pIi0/HXglcC/wJeCiBrHmn+Ei4PZIncSrKSKujIj1EbGB7N/i7RHxhl6LE0DS8ZJOyJfJ+tzvpgf//iPi+8A+SS9IRWcD9/RirMnrOdIdlcfTi3GW1+1BlE69yK40+CeyPu3/2gPxfAJ4GPgx2S+Iy8j6pb8IfCf9+axUV8C1Kfa7gE1rGOdmsubvt4BKep3fo7H+HPCNFOvdwNtS+fOBrwGzZM3/41L509L72bT++V34dzDFkaukei7OFNM302tv/n+nF//+0/Engd3p38CngGf2YqxkF2X8MzBeKOu5OJf78q1BzMyslEHpkjIzs1XmhGFmZqU4YZiZWSlOGGZmVooThpmZleKEYWZmpThhmJlZKf8f3TQGvYgfIUYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Build a forest and compute the feature importances\n",
    "forest = ExtraTreesClassifier(n_estimators=19,\n",
    "                              random_state=32)\n",
    "\n",
    "forest.fit(X_train, y_train)\n",
    "importances = forest.feature_importances_\n",
    "std = np.std([tree.feature_importances_ for tree in forest.estimators_],\n",
    "             axis=0)\n",
    "indices = np.argsort(importances)[::-1]\n",
    "\n",
    "# Print the feature ranking\n",
    "print(\"Feature ranking:\")\n",
    "\n",
    "for f in range(X_train.shape[1]):\n",
    "    print(\"%d. feature %d (%f)\" % (f + 1, indices[f], importances[indices[f]]))\n",
    "\n",
    "# Plot the feature importances of the forest\n",
    "plt.figure()\n",
    "plt.title(\"Feature importances\")\n",
    "plt.bar(range(X_train.shape[1]), importances[indices],\n",
    "       color=\"r\", yerr=std[indices], align=\"center\")\n",
    "plt.xlim([-1, X_train.shape[1]])\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}