modelling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 0. Setup Paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "WORKSPACE_PATH = 'Tensorflow/workspace'\n",
    "SCRIPTS_PATH = 'Tensorflow/scripts'\n",
    "APIMODEL_PATH = 'Tensorflow/models'\n",
    "ANNOTATION_PATH = WORKSPACE_PATH+'/annotations'\n",
    "IMAGE_PATH = WORKSPACE_PATH+'/images'\n",
    "MODEL_PATH = WORKSPACE_PATH+'/models'\n",
    "PRETRAINED_MODEL_PATH = WORKSPACE_PATH+'/pre-trained-models'\n",
    "CONFIG_PATH = MODEL_PATH+'/my_ssd_mobnet/pipeline.config'\n",
    "CHECKPOINT_PATH = MODEL_PATH+'/my_ssd_mobnet/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Create Label Map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = [{'name':'hello', 'id':1},\n",
    "          {'name':'yes', 'id':2},\n",
    "          {'name':'no', 'id':3},\n",
    "          {'name':'thankyou', 'id':4},\n",
    "          {'name':'iloveyou', 'id':5}\n",
    "         ]\n",
    "\n",
    "with open(ANNOTATION_PATH + '\\label_map.pbtxt', 'w') as f:\n",
    "    for label in labels:\n",
    "        f.write('item { \\n')\n",
    "        f.write('\\tname:\\'{}\\'\\n'.format(label['name']))\n",
    "        f.write('\\tid:{}\\n'.format(label['id']))\n",
    "        f.write('}\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Create TF records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully created the TFRecord file: Tensorflow/workspace/annotations/train.record\n",
      "Successfully created the TFRecord file: Tensorflow/workspace/annotations/test.record\n"
     ]
    }
   ],
   "source": [
    "!python {SCRIPTS_PATH + '/generate_tfrecord.py'} -x {IMAGE_PATH + '/train'} -l {ANNOTATION_PATH + '/label_map.pbtxt'} -o {ANNOTATION_PATH + '/train.record'}\n",
    "!python {SCRIPTS_PATH + '/generate_tfrecord.py'} -x{IMAGE_PATH + '/test'} -l {ANNOTATION_PATH + '/label_map.pbtxt'} -o {ANNOTATION_PATH + '/test.record'}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Download TF Models Pretrained Models from Tensorflow Model Zoo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Cloning into 'models'...\n",
      "Updating files:  49% (1372/2773)\n",
      "Updating files:  50% (1387/2773)\n",
      "Updating files:  51% (1415/2773)\n",
      "Updating files:  52% (1442/2773)\n",
      "Updating files:  53% (1470/2773)\n",
      "Updating files:  54% (1498/2773)\n",
      "Updating files:  55% (1526/2773)\n",
      "Updating files:  56% (1553/2773)\n",
      "Updating files:  57% (1581/2773)\n",
      "Updating files:  58% (1609/2773)\n",
      "Updating files:  59% (1637/2773)\n",
      "Updating files:  60% (1664/2773)\n",
      "Updating files:  61% (1692/2773)\n",
      "Updating files:  62% (1720/2773)\n",
      "Updating files:  63% (1747/2773)\n",
      "Updating files:  64% (1775/2773)\n",
      "Updating files:  65% (1803/2773)\n",
      "Updating files:  66% (1831/2773)\n",
      "Updating files:  67% (1858/2773)\n",
      "Updating files:  68% (1886/2773)\n",
      "Updating files:  69% (1914/2773)\n",
      "Updating files:  70% (1942/2773)\n",
      "Updating files:  71% (1969/2773)\n",
      "Updating files:  72% (1997/2773)\n",
      "Updating files:  73% (2025/2773)\n",
      "Updating files:  74% (2053/2773)\n",
      "Updating files:  75% (2080/2773)\n",
      "Updating files:  76% (2108/2773)\n",
      "Updating files:  77% (2136/2773)\n",
      "Updating files:  78% (2163/2773)\n",
      "Updating files:  79% (2191/2773)\n",
      "Updating files:  80% (2219/2773)\n",
      "Updating files:  81% (2247/2773)\n",
      "Updating files:  82% (2274/2773)\n",
      "Updating files:  83% (2302/2773)\n",
      "Updating files:  84% (2330/2773)\n",
      "Updating files:  85% (2358/2773)\n",
      "Updating files:  86% (2385/2773)\n",
      "Updating files:  87% (2413/2773)\n",
      "Updating files:  88% (2441/2773)\n",
      "Updating files:  89% (2468/2773)\n",
      "Updating files:  90% (2496/2773)\n",
      "Updating files:  91% (2524/2773)\n",
      "Updating files:  92% (2552/2773)\n",
      "Updating files:  93% (2579/2773)\n",
      "Updating files:  94% (2607/2773)\n",
      "Updating files:  95% (2635/2773)\n",
      "Updating files:  96% (2663/2773)\n",
      "Updating files:  97% (2690/2773)\n",
      "Updating files:  97% (2707/2773)\n",
      "Updating files:  98% (2718/2773)\n",
      "Updating files:  99% (2746/2773)\n",
      "Updating files: 100% (2773/2773)\n",
      "Updating files: 100% (2773/2773), done.\n"
     ]
    }
   ],
   "source": [
    "!cd Tensorflow && git clone https://github.com/tensorflow/models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Copy Model Config to Training Folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "CUSTOM_MODEL_NAME = 'my_ssd_mobnet' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A subdirectory or file Tensorflow\\workspace\\models\\my_ssd_mobnet already exists.\n",
      "'cp' is not recognized as an internal or external command,\n",
      "operable program or batch file.\n"
     ]
    }
   ],
   "source": [
    "!mkdir {'Tensorflow\\workspace\\models\\\\'+CUSTOM_MODEL_NAME}\n",
    "!cp {PRETRAINED_MODEL_PATH+'/ssd_mobilenet_v2_fpnlite_320x320_coco17_tpu-8/pipeline.config'} {MODEL_PATH+'/'+CUSTOM_MODEL_NAME}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Update Config For Transfer Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from object_detection.utils import config_util\n",
    "from object_detection.protos import pipeline_pb2\n",
    "from google.protobuf import text_format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "CONFIG_PATH = MODEL_PATH+'/'+CUSTOM_MODEL_NAME+'/pipeline.config'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = config_util.get_configs_from_pipeline_file(CONFIG_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model': ssd {\n",
       "   num_classes: 5\n",
       "   image_resizer {\n",
       "     fixed_shape_resizer {\n",
       "       height: 320\n",
       "       width: 320\n",
       "     }\n",
       "   }\n",
       "   feature_extractor {\n",
       "     type: \"ssd_mobilenet_v2_fpn_keras\"\n",
       "     depth_multiplier: 1.0\n",
       "     min_depth: 16\n",
       "     conv_hyperparams {\n",
       "       regularizer {\n",
       "         l2_regularizer {\n",
       "           weight: 3.9999998989515007e-05\n",
       "         }\n",
       "       }\n",
       "       initializer {\n",
       "         random_normal_initializer {\n",
       "           mean: 0.0\n",
       "           stddev: 0.009999999776482582\n",
       "         }\n",
       "       }\n",
       "       activation: RELU_6\n",
       "       batch_norm {\n",
       "         decay: 0.996999979019165\n",
       "         scale: true\n",
       "         epsilon: 0.0010000000474974513\n",
       "       }\n",
       "     }\n",
       "     use_depthwise: true\n",
       "     override_base_feature_extractor_hyperparams: true\n",
       "     fpn {\n",
       "       min_level: 3\n",
       "       max_level: 7\n",
       "       additional_layer_depth: 128\n",
       "     }\n",
       "   }\n",
       "   box_coder {\n",
       "     faster_rcnn_box_coder {\n",
       "       y_scale: 10.0\n",
       "       x_scale: 10.0\n",
       "       height_scale: 5.0\n",
       "       width_scale: 5.0\n",
       "     }\n",
       "   }\n",
       "   matcher {\n",
       "     argmax_matcher {\n",
       "       matched_threshold: 0.5\n",
       "       unmatched_threshold: 0.5\n",
       "       ignore_thresholds: false\n",
       "       negatives_lower_than_unmatched: true\n",
       "       force_match_for_each_row: true\n",
       "       use_matmul_gather: true\n",
       "     }\n",
       "   }\n",
       "   similarity_calculator {\n",
       "     iou_similarity {\n",
       "     }\n",
       "   }\n",
       "   box_predictor {\n",
       "     weight_shared_convolutional_box_predictor {\n",
       "       conv_hyperparams {\n",
       "         regularizer {\n",
       "           l2_regularizer {\n",
       "             weight: 3.9999998989515007e-05\n",
       "           }\n",
       "         }\n",
       "         initializer {\n",
       "           random_normal_initializer {\n",
       "             mean: 0.0\n",
       "             stddev: 0.009999999776482582\n",
       "           }\n",
       "         }\n",
       "         activation: RELU_6\n",
       "         batch_norm {\n",
       "           decay: 0.996999979019165\n",
       "           scale: true\n",
       "           epsilon: 0.0010000000474974513\n",
       "         }\n",
       "       }\n",
       "       depth: 128\n",
       "       num_layers_before_predictor: 4\n",
       "       kernel_size: 3\n",
       "       class_prediction_bias_init: -4.599999904632568\n",
       "       share_prediction_tower: true\n",
       "       use_depthwise: true\n",
       "     }\n",
       "   }\n",
       "   anchor_generator {\n",
       "     multiscale_anchor_generator {\n",
       "       min_level: 3\n",
       "       max_level: 7\n",
       "       anchor_scale: 4.0\n",
       "       aspect_ratios: 1.0\n",
       "       aspect_ratios: 2.0\n",
       "       aspect_ratios: 0.5\n",
       "       scales_per_octave: 2\n",
       "     }\n",
       "   }\n",
       "   post_processing {\n",
       "     batch_non_max_suppression {\n",
       "       score_threshold: 9.99999993922529e-09\n",
       "       iou_threshold: 0.6000000238418579\n",
       "       max_detections_per_class: 100\n",
       "       max_total_detections: 100\n",
       "       use_static_shapes: false\n",
       "     }\n",
       "     score_converter: SIGMOID\n",
       "   }\n",
       "   normalize_loss_by_num_matches: true\n",
       "   loss {\n",
       "     localization_loss {\n",
       "       weighted_smooth_l1 {\n",
       "       }\n",
       "     }\n",
       "     classification_loss {\n",
       "       weighted_sigmoid_focal {\n",
       "         gamma: 2.0\n",
       "         alpha: 0.25\n",
       "       }\n",
       "     }\n",
       "     classification_weight: 1.0\n",
       "     localization_weight: 1.0\n",
       "   }\n",
       "   encode_background_as_zeros: true\n",
       "   normalize_loc_loss_by_codesize: true\n",
       "   inplace_batchnorm_update: true\n",
       "   freeze_batchnorm: false\n",
       " },\n",
       " 'train_config': batch_size: 4\n",
       " data_augmentation_options {\n",
       "   random_horizontal_flip {\n",
       "   }\n",
       " }\n",
       " data_augmentation_options {\n",
       "   random_crop_image {\n",
       "     min_object_covered: 0.0\n",
       "     min_aspect_ratio: 0.75\n",
       "     max_aspect_ratio: 3.0\n",
       "     min_area: 0.75\n",
       "     max_area: 1.0\n",
       "     overlap_thresh: 0.0\n",
       "   }\n",
       " }\n",
       " sync_replicas: true\n",
       " optimizer {\n",
       "   momentum_optimizer {\n",
       "     learning_rate {\n",
       "       cosine_decay_learning_rate {\n",
       "         learning_rate_base: 0.07999999821186066\n",
       "         total_steps: 50000\n",
       "         warmup_learning_rate: 0.026666000485420227\n",
       "         warmup_steps: 1000\n",
       "       }\n",
       "     }\n",
       "     momentum_optimizer_value: 0.8999999761581421\n",
       "   }\n",
       "   use_moving_average: false\n",
       " }\n",
       " fine_tune_checkpoint: \"Tensorflow/workspace/pre-trained-models/ssd_mobilenet_v2_fpnlite_320x320_coco17_tpu-8/checkpoint/ckpt-0\"\n",
       " num_steps: 50000\n",
       " startup_delay_steps: 0.0\n",
       " replicas_to_aggregate: 8\n",
       " max_number_of_boxes: 100\n",
       " unpad_groundtruth_tensors: false\n",
       " fine_tune_checkpoint_type: \"detection\"\n",
       " fine_tune_checkpoint_version: V2,\n",
       " 'train_input_config': label_map_path: \"Tensorflow/workspace/annotations/label_map.pbtxt\"\n",
       " tf_record_input_reader {\n",
       "   input_path: \"Tensorflow/workspace/annotations/train.record\"\n",
       " },\n",
       " 'eval_config': metrics_set: \"coco_detection_metrics\"\n",
       " use_moving_averages: false,\n",
       " 'eval_input_configs': [label_map_path: \"Tensorflow/workspace/annotations/label_map.pbtxt\"\n",
       " shuffle: false\n",
       " num_epochs: 1\n",
       " tf_record_input_reader {\n",
       "   input_path: \"Tensorflow/workspace/annotations/test.record\"\n",
       " }\n",
       " ],\n",
       " 'eval_input_config': label_map_path: \"Tensorflow/workspace/annotations/label_map.pbtxt\"\n",
       " shuffle: false\n",
       " num_epochs: 1\n",
       " tf_record_input_reader {\n",
       "   input_path: \"Tensorflow/workspace/annotations/test.record\"\n",
       " }}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()\n",
    "with tf.io.gfile.GFile(CONFIG_PATH, \"r\") as f:                                                                                                                                                                                                                     \n",
    "    proto_str = f.read()                                                                                                                                                                                                                                          \n",
    "    text_format.Merge(proto_str, pipeline_config)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline_config.model.ssd.num_classes = 5\n",
    "pipeline_config.train_config.batch_size = 4\n",
    "pipeline_config.train_config.fine_tune_checkpoint = PRETRAINED_MODEL_PATH+'/ssd_mobilenet_v2_fpnlite_320x320_coco17_tpu-8/checkpoint/ckpt-0'\n",
    "pipeline_config.train_config.fine_tune_checkpoint_type = \"detection\"\n",
    "pipeline_config.train_input_reader.label_map_path= ANNOTATION_PATH + '/label_map.pbtxt'\n",
    "pipeline_config.train_input_reader.tf_record_input_reader.input_path[:] = [ANNOTATION_PATH + '/train.record']\n",
    "pipeline_config.eval_input_reader[0].label_map_path = ANNOTATION_PATH + '/label_map.pbtxt'\n",
    "pipeline_config.eval_input_reader[0].tf_record_input_reader.input_path[:] = [ANNOTATION_PATH + '/test.record']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_text = text_format.MessageToString(pipeline_config)                                                                                                                                                                                                        \n",
    "with tf.io.gfile.GFile(CONFIG_PATH, \"wb\") as f:                                                                                                                                                                                                                     \n",
    "    f.write(config_text)   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "python Tensorflow/models/research/object_detection/model_main_tf2.py --model_dir=Tensorflow/workspace/models/my_ssd_mobnet --pipeline_config_path=Tensorflow/workspace/models/my_ssd_mobnet/pipeline.config --num_train_steps=10000\n"
     ]
    }
   ],
   "source": [
    "print(\"\"\"python {}/research/object_detection/model_main_tf2.py --model_dir={}/{} --pipeline_config_path={}/{}/pipeline.config --num_train_steps=10000\"\"\".format(APIMODEL_PATH, MODEL_PATH,CUSTOM_MODEL_NAME,MODEL_PATH,CUSTOM_MODEL_NAME))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7. Load Train Model From Checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from object_detection.utils import label_map_util\n",
    "from object_detection.utils import visualization_utils as viz_utils\n",
    "from object_detection.builders import model_builder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load pipeline config and build a detection model\n",
    "configs = config_util.get_configs_from_pipeline_file(CONFIG_PATH)\n",
    "detection_model = model_builder.build(model_config=configs['model'], is_training=False)\n",
    "\n",
    "# Restore checkpoint\n",
    "ckpt = tf.compat.v2.train.Checkpoint(model=detection_model)\n",
    "ckpt.restore(os.path.join(CHECKPOINT_PATH, 'ckpt-8')).expect_partial()\n",
    "\n",
    "@tf.function\n",
    "def detect_fn(image):\n",
    "    image, shapes = detection_model.preprocess(image)\n",
    "    prediction_dict = detection_model.predict(image, shapes)\n",
    "    detections = detection_model.postprocess(prediction_dict, shapes)\n",
    "    return detections"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8. Detect in Real-Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2 \n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "category_index = label_map_util.create_category_index_from_labelmap(ANNOTATION_PATH+'/label_map.pbtxt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup capture\n",
    "cap = cv2.VideoCapture(0)\n",
    "width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))\n",
    "height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "while True: \n",
    "    ret, frame = cap.read()\n",
    "    image_np = np.array(frame)\n",
    "    \n",
    "    input_tensor = tf.convert_to_tensor(np.expand_dims(image_np, 0), dtype=tf.float32)\n",
    "    detections = detect_fn(input_tensor)\n",
    "    \n",
    "    num_detections = int(detections.pop('num_detections'))\n",
    "    detections = {key: value[0, :num_detections].numpy()\n",
    "                  for key, value in detections.items()}\n",
    "    detections['num_detections'] = num_detections\n",
    "\n",
    "    # detection_classes should be ints.\n",
    "    detections['detection_classes'] = detections['detection_classes'].astype(np.int64)\n",
    "\n",
    "    label_id_offset = 1\n",
    "    image_np_with_detections = image_np.copy()\n",
    "\n",
    "    viz_utils.visualize_boxes_and_labels_on_image_array(\n",
    "                image_np_with_detections,\n",
    "                detections['detection_boxes'],\n",
    "                detections['detection_classes']+label_id_offset,\n",
    "                detections['detection_scores'],\n",
    "                category_index,\n",
    "                use_normalized_coordinates=True,\n",
    "                max_boxes_to_draw=5,\n",
    "                min_score_thresh=.5,\n",
    "                agnostic_mode=False)\n",
    "\n",
    "    cv2.imshow('object detection',  cv2.resize(image_np_with_detections, (800, 600)))\n",
    "    \n",
    "    if cv2.waitKey(1) & 0xFF == ord('q'):\n",
    "        cap.release()\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "detections = detect_fn(input_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}