update ciou loss and xyscale

Author: hunglc007

convert_tflite.py

@@ -27,14 +27,6 @@ def representative_data_gen():
     else:
       continue
 
-# def apply_quantization_to_dense(layer):
-#   # print(layer.name)
-#   if isinstance(layer, (tf.keras.layers.Conv2D, tf.keras.layers.BatchNormalization,
-#                         tf.keras.layers.ZeroPadding2D, tf.keras.layers.ReLU)):
-#     print(layer.name)
-#     return tfmot.quantization.keras.quantize_annotate_layer(layer)
-#   return layer
-
 def save_tflite():
   input_layer = tf.keras.layers.Input([FLAGS.input_size, FLAGS.input_size, 3])
   if FLAGS.tiny:
@@ -58,15 +50,6 @@ def save_tflite():
     model.summary()
     utils.load_weights(model, FLAGS.weights)
 
-
-
-  # annotated_model = tf.keras.models.clone_model(
-  #   model,
-  #   clone_function=apply_quantization_to_dense,
-  # )
-  # quant_aware_model = tfmot.quantization.keras.quantize_apply(annotated_model)
-  # quant_aware_model.summary()
-
   converter = tf.lite.TFLiteConverter.from_keras_model(model)
   if FLAGS.quantize_mode == 'int8':
     converter.optimizations = [tf.lite.Optimize.DEFAULT]

core/common.py

@@ -38,6 +38,9 @@ def convolutional(input_layer, filters_shape, downsample=False, activate=True, b
             conv = tf.nn.leaky_relu(conv, alpha=0.1)
         elif activate_type == "mish":
             conv = mish(conv)
+            # conv = softplus(conv)
+            # conv = conv * tf.math.tanh(tf.math.softplus(conv))
+            # conv = conv * tf.tanh(softplus(conv))
             # conv = tf.nn.leaky_relu(conv, alpha=0.1)
             # conv = tfa.activations.mish(conv)
             # conv = conv * tf.nn.tanh(tf.keras.activations.relu(tf.nn.softplus(conv), max_value=20))
@@ -46,9 +49,22 @@ def convolutional(input_layer, filters_shape, downsample=False, activate=True, b
     # if activate == True: conv = tf.keras.layers.ReLU()(conv)
 
     return conv
-
+def softplus(x, threshold = 20.):
+    def f1():
+        return x
+    def f2():
+        return tf.exp(x)
+    def f3():
+        return tf.math.log(1 + tf.exp(x))
+    # mask = tf.greater(x, threshold)
+    # x = tf.exp(x[mask])
+    # return tf.exp(x)
+    return tf.case([(tf.greater(x, tf.constant(threshold)), lambda:f1()), (tf.less(x, tf.constant(-threshold)), lambda:f2())], default=lambda:f3())
+    # return tf.case([(tf.greater(x, threshold), lambda:f1())])
 def mish(x):
     return tf.keras.layers.Lambda(lambda x: x*tf.tanh(tf.math.log(1+tf.exp(x))))(x)
+    # return tf.keras.layers.Lambda(lambda x: softplus(x))(x)
+    # return tf.keras.layers.Lambda(lambda x: x * tf.tanh(softplus(x)))(x)
 
 def residual_block(input_layer, input_channel, filter_num1, filter_num2):
     short_cut = input_layer

core/config.py

@@ -14,9 +14,10 @@
 # Set the class name
 __C.YOLO.CLASSES              = "./data/classes/coco.names"
 __C.YOLO.ANCHORS              = "./data/anchors/coco_anchors.txt"
-__C.YOLO.ANCHORS_TINY              = "./data/anchors/basline_tiny_anchors.txt"
+__C.YOLO.ANCHORS_TINY         = "./data/anchors/basline_tiny_anchors.txt"
 __C.YOLO.STRIDES              = [8, 16, 32]
 __C.YOLO.STRIDES_TINY         = [16, 32]
+__C.YOLO.XYSCALE              = [1.2, 1.1, 1.05]
 __C.YOLO.ANCHOR_PER_SCALE     = 3
 __C.YOLO.IOU_LOSS_THRESH      = 0.5
 
@@ -26,8 +27,8 @@
 
 __C.TRAIN.ANNOT_PATH          = "./data/dataset/val2017.txt"
 __C.TRAIN.BATCH_SIZE          = 4
-# __C.TRAIN.INPUT_SIZE            = [320, 352, 384, 416, 448, 480, 512, 544, 576, 608]
-__C.TRAIN.INPUT_SIZE          = [416]
+__C.TRAIN.INPUT_SIZE            = [320, 352, 384, 416, 448, 480, 512, 544, 576, 608]
+# __C.TRAIN.INPUT_SIZE          = [416]
 __C.TRAIN.DATA_AUG            = True
 __C.TRAIN.LR_INIT             = 1e-3
 __C.TRAIN.LR_END              = 1e-6
@@ -44,7 +45,7 @@
 __C.TEST.INPUT_SIZE           = 416
 __C.TEST.DATA_AUG             = False
 __C.TEST.DECTECTED_IMAGE_PATH = "./data/detection/"
-__C.TEST.SCORE_THRESHOLD      = 0.3
+__C.TEST.SCORE_THRESHOLD      = 0.25
 __C.TEST.IOU_THRESHOLD        = 0.5
 
 

core/utils.py

@@ -2,6 +2,7 @@
 import random
 import colorsys
 import numpy as np
+import tensorflow as tf
 from core.config import cfg
 
 def load_weights_tiny(model, weights_file):
@@ -220,6 +221,40 @@ def bboxes_iou(boxes1, boxes2):
 
     return ious
 
+def bboxes_ciou(boxes1, boxes2):
+
+    boxes1 = np.array(boxes1)
+    boxes2 = np.array(boxes2)
+
+    left = np.maximum(boxes1[..., 0], boxes2[..., 0])
+    up = np.maximum(boxes1[..., 1], boxes2[..., 1])
+    right = np.maximum(boxes1[..., 2], boxes2[..., 2])
+    down = np.maximum(boxes1[..., 3], boxes2[..., 3])
+
+    c = (right - left) * (right - left) + (up - down) * (up - down)
+    iou = bboxes_iou(boxes1, boxes2)
+
+    ax = (boxes1[..., 0] + boxes1[..., 2]) / 2
+    ay = (boxes1[..., 1] + boxes1[..., 3]) / 2
+    bx = (boxes2[..., 0] + boxes2[..., 2]) / 2
+    by = (boxes2[..., 1] + boxes2[..., 3]) / 2
+
+    u = (ax - bx) * (ax - bx) + (ay - by) * (ay - by)
+    d = u/c
+
+    aw = boxes1[..., 2] - boxes1[..., 0]
+    ah = boxes1[..., 3] - boxes1[..., 1]
+    bw = boxes2[..., 2] - boxes2[..., 0]
+    bh = boxes2[..., 3] - boxes2[..., 1]
+
+    ar_gt = bw/bh
+    ar_pred = aw/ah
+
+    ar_loss = 4 / (np.pi * np.pi) * (np.arctan(ar_gt) - np.arctan(ar_pred)) * (np.arctan(ar_gt) - np.arctan(ar_pred))
+    alpha = ar_loss / (1 - iou + ar_loss + 0.000001)
+    ciou_term = d + alpha * ar_loss
+
+    return iou - ciou_term
 
 def nms(bboxes, iou_threshold, sigma=0.3, method='nms'):
     """
@@ -258,7 +293,30 @@ def nms(bboxes, iou_threshold, sigma=0.3, method='nms'):
 
     return best_bboxes
 
-
+def diounms_sort(bboxes, iou_threshold, sigma=0.3, method='nms', beta_nms=0.6):
+    best_bboxes = []
+    return best_bboxes
+def postprocess_bbbox(pred_bbox, XYSCALE, ANCHORS, STRIDES):
+    for i, pred in enumerate(pred_bbox):
+        conv_shape = pred.shape
+        output_size = conv_shape[1]
+        conv_raw_dxdy = pred[:, :, :, :, 0:2]
+        conv_raw_dwdh = pred[:, :, :, :, 2:4]
+        xy_grid = np.meshgrid(np.arange(output_size), np.arange(output_size))
+        xy_grid = np.expand_dims(np.stack(xy_grid, axis=-1), axis=2)  # [gx, gy, 1, 2]
+
+        xy_grid = np.tile(tf.expand_dims(xy_grid, axis=0), [1, 1, 1, 3, 1])
+        xy_grid = xy_grid.astype(np.float)
+
+        # pred_xy = (tf.sigmoid(conv_raw_dxdy) + xy_grid) * STRIDES[i]
+        pred_xy = ((tf.sigmoid(conv_raw_dxdy) * XYSCALE[i]) - 0.5 * (XYSCALE[i] - 1) + xy_grid) * STRIDES[i]
+        # pred_wh = (tf.exp(conv_raw_dwdh) * ANCHORS[i]) * STRIDES[i]
+        pred_wh = (tf.exp(conv_raw_dwdh) * ANCHORS[i])
+        pred[:, :, :, :, 0:4] = tf.concat([pred_xy, pred_wh], axis=-1)
+
+    pred_bbox = [tf.reshape(x, (-1, tf.shape(x)[-1])) for x in pred_bbox]
+    pred_bbox = tf.concat(pred_bbox, axis=0)
+    return pred_bbox
 def postprocess_boxes(pred_bbox, org_img_shape, input_size, score_threshold):
 
     valid_scale=[0, np.inf]

core/yolov4.py

@@ -178,6 +178,32 @@ def bbox_iou(boxes1, boxes2):
 
     return 1.0 * inter_area / union_area
 
+def bbox_ciou(boxes1, boxes2):
+    boxes1_coor = tf.concat([boxes1[..., :2] - boxes1[..., 2:] * 0.5,
+                        boxes1[..., :2] + boxes1[..., 2:] * 0.5], axis=-1)
+    boxes2_coor = tf.concat([boxes2[..., :2] - boxes2[..., 2:] * 0.5,
+                        boxes2[..., :2] + boxes2[..., 2:] * 0.5], axis=-1)
+
+    left = tf.maximum(boxes1_coor[..., 0], boxes2_coor[..., 0])
+    up = tf.maximum(boxes1_coor[..., 1], boxes2_coor[..., 1])
+    right = tf.maximum(boxes1_coor[..., 2], boxes2_coor[..., 2])
+    down = tf.maximum(boxes1_coor[..., 3], boxes2_coor[..., 3])
+
+    c = (right - left) * (right - left) + (up - down) * (up - down)
+    iou = bbox_iou(boxes1, boxes2)
+
+    u = (boxes1[..., 0] - boxes2[..., 0]) * (boxes1[..., 0] - boxes2[..., 0]) + (boxes1[..., 1] - boxes2[..., 1]) * (boxes1[..., 1] - boxes2[..., 1])
+    d = u / c
+
+    ar_gt = boxes2[..., 2] / boxes2[..., 3]
+    ar_pred = boxes1[..., 2] / boxes1[..., 3]
+
+    ar_loss = 4 / (np.pi * np.pi) * (tf.atan(ar_gt) - tf.atan(ar_pred)) * (tf.atan(ar_gt) - tf.atan(ar_pred))
+    alpha = ar_loss / (1 - iou + ar_loss + 0.000001)
+    ciou_term = d + alpha * ar_loss
+
+    return iou - ciou_term
+
 def bbox_giou(boxes1, boxes2):
 
     boxes1 = tf.concat([boxes1[..., :2] - boxes1[..., 2:] * 0.5,
@@ -209,7 +235,6 @@ def bbox_giou(boxes1, boxes2):
 
     return giou
 
-
 def compute_loss(pred, conv, label, bboxes, i=0):
 
     conv_shape  = tf.shape(conv)

detect.py

@@ -24,6 +24,7 @@ def main(_argv):
     else:
         STRIDES = np.array(cfg.YOLO.STRIDES)
         ANCHORS = utils.get_anchors(cfg.YOLO.ANCHORS, FLAGS.tiny)
+    XYSCALE = cfg.YOLO.XYSCALE
     input_size = FLAGS.size
     image_path = FLAGS.image
 
@@ -69,26 +70,9 @@ def main(_argv):
         interpreter.invoke()
         pred_bbox = [interpreter.get_tensor(output_details[i]['index']) for i in range(len(output_details))]
 
-    for i, pred in enumerate(pred_bbox):
-        conv_shape = pred.shape
-        output_size = conv_shape[1]
-        conv_raw_dxdy = pred[:, :, :, :, 0:2]
-        conv_raw_dwdh = pred[:, :, :, :, 2:4]
-        xy_grid = np.meshgrid(np.arange(output_size), np.arange(output_size))
-        xy_grid = np.expand_dims(np.stack(xy_grid, axis=-1), axis=2)  # [gx, gy, 1, 2]
-
-        xy_grid = np.tile(tf.expand_dims(xy_grid, axis=0), [1, 1, 1, 3, 1])
-        xy_grid = xy_grid.astype(np.float)
-
-        pred_xy = (tf.sigmoid(conv_raw_dxdy) + xy_grid) * STRIDES[i]
-        # pred_wh = (tf.exp(conv_raw_dwdh) * ANCHORS[i]) * STRIDES[i]
-        pred_wh = (tf.exp(conv_raw_dwdh) * ANCHORS[i])
-        pred[:, :, :, :, 0:4] = tf.concat([pred_xy, pred_wh], axis=-1)
-
-    pred_bbox = [tf.reshape(x, (-1, tf.shape(x)[-1])) for x in pred_bbox]
-    pred_bbox = tf.concat(pred_bbox, axis=0)
-    bboxes = utils.postprocess_boxes(pred_bbox, original_image_size, input_size, 0.3)
-    bboxes = utils.nms(bboxes, 0.45, method='nms')
+    pred_bbox = utils.postprocess_bbbox(pred_bbox, XYSCALE, ANCHORS, STRIDES)
+    bboxes = utils.postprocess_boxes(pred_bbox, original_image_size, input_size, 0.25)
+    bboxes = utils.nms(bboxes, 0.213, method='nms')
 
     image = utils.draw_bbox(original_image, bboxes)
     image = Image.fromarray(image)

requirements-gpu.txt

@@ -5,4 +5,5 @@ tqdm
 absl-py
 matplotlib
 easydict
+pillow
 tensorflow_addons==0.9.1

requirements.txt

@@ -5,4 +5,5 @@ tensorflow==2.1.0
 absl-py
 easydict
 matplotlib
+pillow
 tensorflow_addons==0.9.1