support RLE in convert_to_coco_dict

Summary:
fix facebookresearch#200
Pull Request resolved: fairinternal/detectron2#391

Reviewed By: rbgirshick

Differential Revision: D20260189

Pulled By: ppwwyyxx

fbshipit-source-id: 690b497e45dda9a3dc9696dc991aac7bf482de0b

Author: ppwwyyxx

detectron2/data/datasets/coco.py

@@ -1,20 +1,19 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
-import io
-import logging
 import contextlib
-import os
 import datetime
+import io
 import json
+import logging
 import numpy as np
-
-from PIL import Image
-
-from fvcore.common.timer import Timer
-from detectron2.structures import BoxMode, PolygonMasks, Boxes
+import os
+import pycocotools.mask as mask_util
 from fvcore.common.file_io import PathManager, file_lock
+from fvcore.common.timer import Timer
+from PIL import Image
 
+from detectron2.structures import Boxes, BoxMode, PolygonMasks
 
-from .. import MetadataCatalog, DatasetCatalog
+from .. import DatasetCatalog, MetadataCatalog
 
 """
 This file contains functions to parse COCO-format annotations into dicts in "Detectron2 format".
@@ -333,8 +332,13 @@ def convert_to_coco_dict(dataset_name):
                 # Computing areas for instances by counting the pixels
                 segmentation = annotation["segmentation"]
                 # TODO: check segmentation type: RLE, BinaryMask or Polygon
-                polygons = PolygonMasks([segmentation])
-                area = polygons.area()[0].item()
+                if isinstance(segmentation, list):
+                    polygons = PolygonMasks([segmentation])
+                    area = polygons.area()[0].item()
+                elif isinstance(segmentation, dict):  # RLE
+                    area = mask_util.area(segmentation)
+                else:
+                    raise TypeError(f"Unknown segmentation type {type(segmentation)}!")
             else:
                 # Computing areas using bounding boxes
                 bbox_xy = BoxMode.convert(bbox, BoxMode.XYWH_ABS, BoxMode.XYXY_ABS)