fix overflow error and loss function / add runner.test() method - now…

… available

config.py

@@ -1,14 +1,17 @@
 # Configuration for Model Tuning
 config = {
+	#Dataset
+	"train_data_xml" : "svt1/train.xml",
+	"test_data_xml" : "svt1/test.xml" ,
     #image size
     "image_size" : 300.0,
     ##################
     # In Learning
     ##################
     # Restoring variables from ckpt file
-    "model_dir" : "saved/model1",
+    "saved_dir" : "saved",
     # Random images Batch size in training
-    "batch_size" : 8,
+    "batch_size" : 1,
     # Learning rate until step 4000, 180000, 240000, and after
     "learning_rate_list" : [8e-4, 1e-3, 1e-4, 1e-5] ,
     # Whether Doing Batch Normalization in training phase
@@ -31,4 +34,4 @@
 
 
 
-}
+}

main.py

@@ -1,23 +1,34 @@
-from config import config
-from shintb import graph_drawer ,svt_data_loader, default_box_control, runner
-
 import tensorflow as tf
 
+from config import config
+from shintb import graph_drawer, svt_data_loader, default_box_control, runner, output_drawer
+
 flags = tf.app.flags
 FLAGS = flags.FLAGS
 
 graphdrawer = graph_drawer.GraphDrawer(config)
 
-dataloader = svt_data_loader.SVTDataLoader('./svt1/train.xml', './svt1/test.xml')
+dataloader = svt_data_loader.SVTDataLoader(config["train_data_xml"], config['test_data_xml'])
 
 dbcontrol = default_box_control.DefaultBoxControl(config, graphdrawer)
 
-runner = runner.Runner(config, graphdrawer, dataloader, dbcontrol)
+outputdrawer = output_drawer.OutputDrawer(config, dbcontrol)
+
+runner = runner.Runner(config, graphdrawer, dataloader, dbcontrol, outputdrawer)
 
 if __name__ == "__main__":
-	flags.DEFINE_string("mode", "train", "train, image")
+	flags.DEFINE_string("mode", "train", "train,test ,image")
+	flags.DEFINE_string("jobname", None, "job name for saving ckpt file")
+	flags.DEFINE_integer("iter", 100000, "iteration for job")
 
 	if FLAGS.mode == "train":
-		runner.train()
+		if FLAGS.jobname ==None :
+			raise FileNotFoundError("jobname 을 입력하지 않았습니다")
+		else :
+			runner.train(FLAGS.jobname, FLAGS.iter)
+
+	elif FLAGS.mode == "test":
+		runner.test(FLAGS.iter)
+
 	elif FLAGS.mode == "image":
-		runner.image()
+		runner.image()

shintb/default_box_control.py

@@ -1,5 +1,6 @@
 import shintb.utils.box_calculation as boxcal
 import numpy as np
+import tensorflow as tf
 
 
 class DefaultBoxControl:
@@ -67,7 +68,10 @@ def initialize_default_boxes(self):
 
         return boxes
 
+
+    # pred_conf : [?, 23280, 2]
     def calculate_pos_neg_trueloc(self, pred_conf, gtboxes):
+        print("calculate_pos_neg_trueloc START")
         c = self.config
 
         pos_neg_trueloc = [None for i in range(c["batch_size"])]
@@ -81,14 +85,12 @@ def calculate_pos_neg_trueloc(self, pred_conf, gtboxes):
 
         positive, negative, _true_conf ,true_loc = [np.stack(m) for m in zip(*pos_neg_trueloc)]
 
-
+        print("calculate_pos_neg_trueloc END")
         return positive, negative, true_loc
 
-
-
-
+    # pred_conf_i : [23280, 2]
     def matching_dbboxes_gtboxes_in_batch_i(self, pred_conf_i, gtboxes_i):
-
+        print("matching_dbboxes_gtboxes_in_batch_i START")
         c = self.config
 
         #default box들의 좌표와 번째 수를 나열해놓은 리스트
@@ -204,6 +206,7 @@ def matching_dbboxes_gtboxes_in_batch_i(self, pred_conf_i, gtboxes_i):
         negative_max = positive_count * c["neg/pos"]
         negative_count = 0
 
+        # pred_conf_i : [ 23280, 2]
         top_confidences = self.get_top_confidences(pred_conf_i, negative_max)
         # top confidence 들의 index list 를 반환
 
@@ -229,22 +232,48 @@ def matching_dbboxes_gtboxes_in_batch_i(self, pred_conf_i, gtboxes_i):
 
         #print("%i positives" % positive_count)
         #print("%i/%i negatives" % (negative_count, negative_max))
-
+        print("matching_dbboxes_gtboxes_in_batch_i END")
         return matches
 
-    def get_top_confidences(self, pred_conf, negative_max):
+    #pred_conf_i : [23280, 2]
+    def get_top_confidences(self, pred_conf_i, negative_max):
+        print("get_top_confidences START")
         confidences = []
 
+        # pred_conf_i : [23280, 2]
+        #logits : [c1, c2]
+
+        # 오버플로우 방지!
+        def softmax_without_overflow(a, reduce_axis=1) :
+            c = np.max(a, reduce_axis)
+            c = np.reshape(c, [-1,1])
+            exp_a = np.exp(a - c)
+            sum_exp_a = a.sum(reduce_axis)
+            sum_exp_a = np.reshape(sum_exp_a, [-1, 1])
+
+            return exp_a / (sum_exp_a + 1.0e-4)
+
+        pred_conf_i_softmax =  softmax_without_overflow(pred_conf_i)
+        # pred_conf_i_softmax = tf.nn.softmax(pred_conf_i, -1)
+
+        for probs in pred_conf_i_softmax :
+            top_confidence = np.amax(probs)
+            confidences.append(top_confidence)
+
+        '''
         for logits in pred_conf:
-            probs = np.exp(logits) / np.add(np.sum(np.exp(logits)), 1e-3)
+            #probs = (1.0/np.exp(-logits)) / np.add(np.sum((1.0/np.exp(-logits))), 1e-3)
+            probs = tf.nn.softmax(logits)
+            # probs : [softmax_c1, softmax_c2]
             top_label = np.amax(probs)
             confidences.append(top_label)
-
+        '''
         # top_confidences = sorted(confidences, key=lambda tup: tup[1])[::-1]
 
         k = min(negative_max, len(confidences))
         top_confidences = np.argpartition(np.asarray(confidences), -k)[-k:]
 
+        print("get_top_confidences END")
         return top_confidences
 
     # prepare_feed  (이미지 하나에 대하여)
@@ -255,7 +284,7 @@ def get_top_confidences(self, pred_conf, negative_max):
     # a_true_labels (23280, )
     # a_true_locs (23280, 4)
     def prepare_pos_neg_trueloc_in_matches(self, matches):
-
+        print("prepare_pos_neg_trueloc_in_matches START")
         c = self.config
 
         positives_list = []
@@ -298,6 +327,7 @@ def prepare_pos_neg_trueloc_in_matches(self, matches):
         a_true_labels = np.asarray(true_labels_list)
         a_true_locs = np.asarray(true_locs_list)
 
+        print("prepare_pos_neg_trueloc_in_matches END  ")
         return a_positives, a_negatives, a_true_labels, a_true_locs
 
 

shintb/graph_drawer.py

@@ -16,6 +16,7 @@ def __init__(self, config):
         self.draw_loss_function()
         self.draw_optimizer()
         self.init_op = self.draw_init_op()
+        self.draw_summary_op()
         print("Complete !!!")
 
     def draw_placeholders(self):
@@ -107,23 +108,30 @@ def draw_predictions(self):
 
         with tf.name_scope(name="predictions") as scope :
             self.pred_conf , self.pred_loc = tf.split(pred_boxes, [2,4], axis = 2)
-            #self.pred_confs : [?, 23280 , 2]
-            #self.pred_locs : [? ,23280, 4]
+            #self.pred_conf : [?, 23280 , 2]
+            #self.pred_loc : [? ,23280, 4]
 
         print("Done!")
 
     def draw_loss_function(self):
         print(">>>Draw loss function ...", end=" ")
-        text_conf , background_conf = tf.split(self.pred_conf, 2, 2)
+        # [?, 23280, 2]
+        pred_conf_softmax =  tf.nn.softmax(self.pred_conf, -1)
+        # [?, 23280, 1], [?, 23280, 1]
+        text_conf , background_conf = tf.split(pred_conf_softmax, 2, 2)
+        # [?, 23280]
         text_conf = tf.reshape(text_conf, [-1, text_conf.get_shape().as_list()[1]] )
+        # [?, 23280]
         background_conf = tf.reshape(background_conf, [-1, background_conf.get_shape().as_list()[1]])
-        self.conf_loss =  - tf.reduce_sum(tf.log(tf.nn.softmax(text_conf))*self.positive_ph) - tf.reduce_sum(tf.log(tf.nn.softmax(background_conf))*self.negative_ph)
+        self.conf_loss =  - tf.reduce_sum(tf.log(text_conf)*self.positive_ph) - tf.reduce_sum(tf.log(background_conf)*self.negative_ph)
 
         loc_loss = tf.reduce_sum(gc.smooth_l1(self.pred_loc - self.true_loc_ph), reduction_indices= 2) * self.positive_ph
         self.loc_loss = tf.reduce_sum(loc_loss, reduction_indices= 1) / (tf.reduce_sum(self.positive_ph, reduction_indices = 1) + 1e-5)
         # 1e-5 는 분모가 0일 경우를 방지
 
         self.total_loss = tf.reduce_mean(self.conf_loss + self.loc_loss)
+
+
         print("Done!")
 
     def draw_optimizer(self):
@@ -137,3 +145,13 @@ def draw_optimizer(self):
     def draw_init_op(self):
         init_op = tf.global_variables_initializer()
         return init_op
+
+    def draw_summary_op(self):
+        print(">>>Draw summary op ...", end=" ")
+        tf.summary.scalar("loss/conf_loss", tf.reshape(tf.reduce_sum(self.conf_loss), shape=[]))
+        tf.summary.scalar("loss/loc_loss", tf.reshape(tf.reduce_sum(self.loc_loss), shape=[]))
+        tf.summary.scalar("loss/total_loss",  tf.reshape(self.total_loss, shape=[]))
+
+        self.summaries = tf.summary.merge_all()
+        print("Done!")
+

shintb/output_drawer.py

@@ -0,0 +1,154 @@
+import numpy as np
+import colorsys
+import cv2
+
+import shintb.utils.box_calculation as boxcal
+
+
+class OutputDrawer:
+    def __init__(self, config, dbcontrol):
+        self.config = config
+        self.dbcontrol = dbcontrol
+
+
+    # for one image
+    # pred_conf : [23830, 2]
+    # pred_loc : [23830, 4]
+    #
+
+    def format_output(self, pred_conf, pred_loc, boxes=None, confidences=None):
+
+        c = self.config
+
+        if boxes is None:
+
+            #[6, x, y, 12] shape list
+
+            boxes = [
+                [[[None for i in range(c["layer_boxes"][o])] for x in range(self.dbcontrol.out_shape[o][1])] for y in
+                 range(self.dbcontrol.out_shape[o][2])]
+                for o in range(len(c["layer_boxes"]))]
+
+        if confidences is None:
+
+            confidences = []
+
+        index = 0  # 1 index -> 1 box (among 23280 boxes)
+
+        # 6
+        for o_i in range(len(c["layer_boxes"])):
+            # x
+            for x in range(self.dbcontrol.out_shape[o_i][2]):
+                # y
+                for y in range(self.dbcontrol.out_shape[o_i][1]):
+                    # 12
+                    for i in range(c["layer_boxes"][o_i]):
+
+                        # for one image
+                        # pred_conf : [23830, 2] (logits)
+                        # pred_loc : [23830, 4]
+
+                        diffs = pred_loc[index] #[dx,dy,dw,dh]
+                        original = self.dbcontrol.default_boxes[o_i][x][y][i] #[x,y,w,h]
+
+                        c_x = original[0] + original[2] * diffs[0]   # x+ w*dx
+                        c_y = original[1] + original[3] * diffs[1]  #y + y*dy
+                        w = original[2] * np.exp(diffs[2])     # w * exp(dw)
+                        h = original[3] * np.exp(diffs[3])   # h * exp(dh)
+
+                        boxes[o_i][x][y][i] = [c_x, c_y, w, h]
+                        logits = pred_conf[index]  # [c1, c2]
+                        # if np.argmax(logits) != classes+1:
+                        info = ([o_i, x, y, i],
+                                np.amax(np.exp(logits) / (np.sum(np.exp(logits)) + 1e-3)),
+                                np.argmax(logits))
+                        # indices, max probability, corresponding label
+
+                        # if len(confidences) < index+1:
+                        # 	confidences.append(info)
+                        # else:
+                        # 	confidences[index] = info
+                        # else:
+                        #    logits = pred_conf[index][:-1]
+                        #    confidences.append(([o_i, x, y, i], np.amax(np.exp(logits) / (np.sum(np.exp(logits)) + 1e-3)),
+                        #                        np.argmax(logits)))
+                        confidences.append(info)
+                        index += 1
+
+        # sorted_confidences = sorted(confidences, key=lambda tup: tup[1])[::-1]
+
+        return boxes, confidences
+
+
+    def draw_outputs(self, img, boxes, confidences, wait=1):
+        I = img * 255.0
+
+        # nms = non_max_suppression_fast(np.asarray(filtered_boxes), 1.00)
+        picks = self.postprocess_boxes(boxes, confidences)
+
+        print("PICKED BOXES INFO :", picks)
+
+        for box, conf, top_label in picks:  # [filtered[i] for i in picks]:
+            if top_label != 1:
+                # print("%f: %s %s" % (conf, coco.i2name[top_label], box))
+
+                c = colorsys.hsv_to_rgb(((top_label * 17) % 255) / 255.0, 1.0, 1.0)
+                c = tuple([255 * c[i] for i in range(3)])
+
+        I = cv2.cvtColor(I.astype(np.uint8), cv2.COLOR_RGB2BGR)
+
+        for box, conf, top_label in picks :
+            x, y, w, h = box[0] ,box[1], box[2], box[3]
+            rect_start = (x,y)
+            rect_end = (x+w, y+h)
+            I = cv2.rectangle(I, rect_start, rect_end, (255, 0, 0) , 5 )
+
+            print("Textboxes information!")
+            print("rect_start : ", rect_start , "// rect_end :", rect_end)
+            print("confidence: ", conf)
+
+        #doing GOOD #I = cv2.rectangle(I, (10,10), (100,100), (255,0,0), 5) #test color
+        cv2.imshow("outputs", I )
+        cv2.waitKey(wait)
+
+
+
+    def basic_nms(self, boxes, thres=0.45):
+        re = []
+
+        def pass_nms(c, lab):
+            for box_, conf_, top_label_ in re :
+                #if lab == top_label_ and boxcal.calc_jaccard(c, box_) > thres:
+                if lab == 0 and boxcal.calc_jaccard(c, box_) < thres:
+                    return False
+            return True
+
+
+
+        for box, conf, top_label in boxes:
+            # top_label = 0 : text // top_label=1 : background
+            if top_label != 1 and pass_nms(box, top_label):
+                re.append((box, conf, top_label))
+
+                # re.append(index)
+                if len(re) >= 200:
+                    break
+
+        return re  #[(corneredbox,conf,top_label), ...]
+
+
+    # center to corner process
+    def postprocess_boxes(self, boxes, confidences, min_conf=0.001, nms=0.45):
+        filtered = []
+
+        for box, conf, top_label in confidences:
+            if conf >= min_conf:
+                coords = boxes[box[0]][box[1]][box[2]][box[3]]
+                coords = boxcal.center2cornerbox(coords)
+
+                filtered.append((coords, conf, top_label))
+
+        print("FILTERED BOXES INFO :", filtered)
+
+        return self.basic_nms(filtered, nms)
+