Python pycocotools.cocoeval.COCOeval() Examples

def evaluate(self, path_to_results_dir: str, image_ids: List[str], bboxes: List[List[float]], classes: List[int], probs: List[float]) -> Tuple[float, str]:
        self._write_results(path_to_results_dir, image_ids, bboxes, classes, probs)

        annType = 'bbox'
        path_to_coco_dir = os.path.join(self._path_to_data_dir, 'COCO')
        path_to_annotations_dir = os.path.join(path_to_coco_dir, 'annotations')
        path_to_annotation = os.path.join(path_to_annotations_dir, 'instances_val2017.json')

        cocoGt = COCO(path_to_annotation)
        cocoDt = cocoGt.loadRes(os.path.join(path_to_results_dir, 'results.json'))

        cocoEval = COCOeval(cocoGt, cocoDt, annType)
        cocoEval.evaluate()
        cocoEval.accumulate()

        original_stdout = sys.stdout
        string_stdout = StringIO()
        sys.stdout = string_stdout
        cocoEval.summarize()
        sys.stdout = original_stdout

        mean_ap = cocoEval.stats[0].item()  # stats[0] records AP@[0.5:0.95]
        detail = string_stdout.getvalue()

        return mean_ap, detail 

def update(self, coco_eval):
        if coco_eval is None:
            return

        assert isinstance(coco_eval, (COCOeval, denseposeCOCOeval))
        s = coco_eval.stats
        iou_type = coco_eval.params.iouType
        res = self.results[iou_type]
        metrics = COCOResults.METRICS[iou_type]
        if iou_type == 'uv':
            idx_map = [0, 1, 6, 11, 12]
            for idx, metric in enumerate(metrics):
                res[metric] = s[idx_map[idx]]
        else:
            for idx, metric in enumerate(metrics):
                res[metric] = s[idx] 

def __init__(self, groundtruth=None, detections=None, agnostic_mode=False,
               iou_type='bbox'):
    """COCOEvalWrapper constructor.

    Note that for the area-based metrics to be meaningful, detection and
    groundtruth boxes must be in image coordinates measured in pixels.

    Args:
      groundtruth: a coco.COCO (or coco_tools.COCOWrapper) object holding
        groundtruth annotations
      detections: a coco.COCO (or coco_tools.COCOWrapper) object holding
        detections
      agnostic_mode: boolean (default: False).  If True, evaluation ignores
        class labels, treating all detections as proposals.
      iou_type: IOU type to use for evaluation. Supports `bbox` or `segm`.
    """
    cocoeval.COCOeval.__init__(self, groundtruth, detections,
                               iouType=iou_type)
    if agnostic_mode:
      self.params.useCats = 0 

def after_run(self, sess):
    print("Detection Finished ...")

    # for item in self.detection:
    #   print(item)

    if len(self.detection) > 0:
      annotation_file = os.path.join(
        DATASET_DIR,
        "annotations",
        "instances_" + DATASET_META + ".json")
      coco = COCO(annotation_file)

      coco_results = coco.loadRes(self.detection)

      # DETECTION_FILE = "/home/ubuntu/data/mscoco/results/SSD_512x512_score/detections_minival_ssd512_results.json"
      # coco_results = coco.loadRes(DETECTION_FILE)

      cocoEval = COCOeval(coco, coco_results, "bbox")
      cocoEval.params.imgIds = self.image_ids
      cocoEval.evaluate()
      cocoEval.accumulate()
      cocoEval.summarize()
    else:
      print("Found no valid detection. Consider re-train your model.") 

def evaluate(self, path_to_results_dir: str, image_ids: List[str], bboxes: List[List[float]], classes: List[int], probs: List[float]) -> Tuple[float, str]:
        self._write_results(path_to_results_dir, image_ids, bboxes, classes, probs)

        annType = 'bbox'
        path_to_coco_dir = os.path.join(self._path_to_data_dir, 'COCO')
        path_to_annotations_dir = os.path.join(path_to_coco_dir, 'annotations')
        path_to_annotation = os.path.join(path_to_annotations_dir, 'instances_val2017.json')

        cocoGt = COCO(path_to_annotation)
        cocoDt = cocoGt.loadRes(os.path.join(path_to_results_dir, 'results.json'))

        cocoEval = COCOeval(cocoGt, cocoDt, annType)
        cocoEval.evaluate()
        cocoEval.accumulate()

        original_stdout = sys.stdout
        string_stdout = StringIO()
        sys.stdout = string_stdout
        cocoEval.summarize()
        sys.stdout = original_stdout

        mean_ap = cocoEval.stats[0].item()  # stats[0] records AP@[0.5:0.95]
        detail = string_stdout.getvalue()

        return mean_ap, detail 

def evaluate_predictions_on_coco(
    coco_gt, coco_results, json_result_file, iou_type="bbox"
):
    import json

    with open(json_result_file, "w") as f:
        json.dump(coco_results, f)

    from pycocotools.coco import COCO
    from pycocotools.cocoeval import COCOeval

    coco_dt = coco_gt.loadRes(str(json_result_file)) if coco_results else COCO()

    # coco_dt = coco_gt.loadRes(coco_results)
    coco_eval = COCOeval(coco_gt, coco_dt, iou_type)
    coco_eval.evaluate()
    coco_eval.accumulate()
    coco_eval.summarize()
    return coco_eval 

def evaluate(self, result_json, cls_ids, image_ids):
        from pycocotools.cocoeval import COCOeval

        if self._split == "testdev":
            return None

        coco = self._coco

        eval_ids = [self._eval_ids[image_id] for image_id in image_ids]
        cat_ids = [self._cls2coco[cls_id] for cls_id in cls_ids]

        coco_dets = coco.loadRes(result_json)
        coco_eval = COCOeval(coco, coco_dets, "bbox")
        coco_eval.params.imgIds = eval_ids
        coco_eval.params.catIds = cat_ids
        coco_eval.evaluate()
        coco_eval.accumulate()
        coco_eval.summarize()
        return coco_eval.stats[0], coco_eval.stats[12:] 

def evaluate_predictions_on_coco(
    coco_gt, coco_results, json_result_file, iou_type="bbox"
):
    import json

    with open(json_result_file, "w") as f:
        json.dump(coco_results, f)

    from pycocotools.coco import COCO
    from pycocotools.cocoeval import COCOeval

    coco_dt = coco_gt.loadRes(str(json_result_file)) if coco_results else COCO()

    # coco_dt = coco_gt.loadRes(coco_results)
    coco_eval = COCOeval(coco_gt, coco_dt, iou_type)
    coco_eval.evaluate()
    coco_eval.accumulate()
    coco_eval.summarize()
    return coco_eval 

def summarize(predictions, annotations_file, img_list, mask=False):
    msk_map = 0
    with tempfile.NamedTemporaryFile("w") as fid:
        json.dump(predictions, fid)
        fid.flush()

        # Detection
        gt = COCO(annotations_file, img_list)
        pred = gt.loadRes(fid.name)
        pred_eval = COCOeval(gt, pred, "bbox")
        pred_eval.evaluate()
        pred_eval.accumulate()
        pred_eval.summarize()
        det_map = pred_eval.stats[0]

        if mask:
            pred_eval = COCOeval(gt, pred, "segm")
            pred_eval.evaluate()
            pred_eval.accumulate()
            pred_eval.summarize()
            msk_map = pred_eval.stats[0]

    return det_map, msk_map 

def evaluate_predictions_on_coco(coco_gt, coco_results, json_result_file, iou_type="bbox"):
    if iou_type != "uv":
        with open(json_result_file, "w") as f:
            json.dump(coco_results, f)
        coco_dt = coco_gt.loadRes(str(json_result_file)) if coco_results else COCO()
        # coco_dt = coco_gt.loadRes(coco_results)
        coco_eval = COCOeval(coco_gt, coco_dt, iou_type)
        coco_eval.evaluate()
    else:
        calc_mode = 'GPSm' if cfg.UVRCNN.GPSM_ON else 'GPS'
        pkl_result_file = json_result_file.replace('.json', '.pkl')
        with open(pkl_result_file, 'wb') as f:
            pickle.dump(coco_results, f, 2)
        if cfg.TEST.DATASETS[0].find('test') > -1:
            return
        eval_data_dir = cfg.DATA_DIR + '/DensePoseData/eval_data/'
        coco_dt = coco_gt.loadRes(coco_results)
        test_sigma = 0.255
        coco_eval = denseposeCOCOeval(eval_data_dir, coco_gt, coco_dt, iou_type, test_sigma)
        coco_eval.evaluate(calc_mode=calc_mode)
    coco_eval.accumulate()
    if iou_type == "bbox":
        _print_detection_eval_metrics(coco_gt, coco_eval)
    coco_eval.summarize()
    return coco_eval 

def evaluate_predictions_on_coco(
    coco_gt, coco_results, json_result_file, iou_type="bbox"
):
    import json

    with open(json_result_file, "w") as f:
        json.dump(coco_results, f)

    from pycocotools.cocoeval import COCOeval

    coco_dt = coco_gt.loadRes(str(json_result_file))
    # coco_dt = coco_gt.loadRes(coco_results)
    coco_eval = COCOeval(coco_gt, coco_dt, iou_type)
    coco_eval.evaluate()
    coco_eval.accumulate()
    coco_eval.summarize()
    return coco_eval 

def __init__(self, groundtruth=None, detections=None, agnostic_mode=False,
               iou_type='bbox'):
    """COCOEvalWrapper constructor.

    Note that for the area-based metrics to be meaningful, detection and
    groundtruth boxes must be in image coordinates measured in pixels.

    Args:
      groundtruth: a coco.COCO (or coco_tools.COCOWrapper) object holding
        groundtruth annotations
      detections: a coco.COCO (or coco_tools.COCOWrapper) object holding
        detections
      agnostic_mode: boolean (default: False).  If True, evaluation ignores
        class labels, treating all detections as proposals.
      iou_type: IOU type to use for evaluation. Supports `bbox` or `segm`.
    """
    cocoeval.COCOeval.__init__(self, groundtruth, detections,
                               iouType=iou_type)
    if agnostic_mode:
      self.params.useCats = 0 

def __init__(self, groundtruth=None, detections=None, agnostic_mode=False,
               iou_type='bbox'):
    """COCOEvalWrapper constructor.

    Note that for the area-based metrics to be meaningful, detection and
    groundtruth boxes must be in image coordinates measured in pixels.

    Args:
      groundtruth: a coco.COCO (or coco_tools.COCOWrapper) object holding
        groundtruth annotations
      detections: a coco.COCO (or coco_tools.COCOWrapper) object holding
        detections
      agnostic_mode: boolean (default: False).  If True, evaluation ignores
        class labels, treating all detections as proposals.
      iou_type: IOU type to use for evaluation. Supports `bbox` or `segm`.
    """
    cocoeval.COCOeval.__init__(self, groundtruth, detections,
                               iouType=iou_type)
    if agnostic_mode:
      self.params.useCats = 0 

def __init__(self, groundtruth=None, detections=None, agnostic_mode=False,
               iou_type='bbox'):
    """COCOEvalWrapper constructor.

    Note that for the area-based metrics to be meaningful, detection and
    groundtruth boxes must be in image coordinates measured in pixels.

    Args:
      groundtruth: a coco.COCO (or coco_tools.COCOWrapper) object holding
        groundtruth annotations
      detections: a coco.COCO (or coco_tools.COCOWrapper) object holding
        detections
      agnostic_mode: boolean (default: False).  If True, evaluation ignores
        class labels, treating all detections as proposals.
      iou_type: IOU type to use for evaluation. Supports `bbox` or `segm`.
    """
    cocoeval.COCOeval.__init__(self, groundtruth, detections,
                               iouType=iou_type)
    if agnostic_mode:
      self.params.useCats = 0 

def evaluate_predictions_on_coco(
    coco_gt, coco_results, json_result_file, iou_type="bbox"
):
    import json

    with open(json_result_file, "w") as f:
        json.dump(coco_results, f)

    from pycocotools.coco import COCO
    from pycocotools.cocoeval import COCOeval

    coco_dt = coco_gt.loadRes(str(json_result_file)) if coco_results else COCO()

    # coco_dt = coco_gt.loadRes(coco_results)
    coco_eval = COCOeval(coco_gt, coco_dt, iou_type)
    coco_eval.evaluate()
    coco_eval.accumulate()
    coco_eval.summarize()
    return coco_eval 

def __init__(self, groundtruth=None, detections=None, agnostic_mode=False,
               iou_type='bbox'):
    """COCOEvalWrapper constructor.

    Note that for the area-based metrics to be meaningful, detection and
    groundtruth boxes must be in image coordinates measured in pixels.

    Args:
      groundtruth: a coco.COCO (or coco_tools.COCOWrapper) object holding
        groundtruth annotations
      detections: a coco.COCO (or coco_tools.COCOWrapper) object holding
        detections
      agnostic_mode: boolean (default: False).  If True, evaluation ignores
        class labels, treating all detections as proposals.
      iou_type: IOU type to use for evaluation. Supports `bbox` or `segm`.
    """
    cocoeval.COCOeval.__init__(self, groundtruth, detections,
                               iouType=iou_type)
    if agnostic_mode:
      self.params.useCats = 0 

def _update(self):
        """Use coco to get real scores. """
        import json
        try:
            with open(self._filename, 'w') as f:
                json.dump(self._results, f)
        except IOError as e:
            raise RuntimeError("Unable to dump json file, ignored. What(): {}".format(str(e)))

        pred = self.dataset.coco.loadRes(self._filename)
        gt = self.dataset.coco
        # lazy import pycocotools
        from ...data.mscoco.utils import try_import_pycocotools
        try_import_pycocotools()
        from pycocotools.cocoeval import COCOeval
        coco_eval = COCOeval(gt, pred, 'keypoints')
        coco_eval.params.useSegm = None
        coco_eval.evaluate()
        coco_eval.accumulate()
        coco_eval.summarize()
        self._coco_eval = coco_eval
        return coco_eval 

def evaluate_predictions_on_coco(
    coco_gt, coco_results, json_result_file, iou_type="bbox"
):
    import json

    with open(json_result_file, "w") as f:
        json.dump(coco_results, f)

    from pycocotools.coco import COCO
    from pycocotools.cocoeval import COCOeval

    coco_dt = coco_gt.loadRes(str(json_result_file)) if coco_results else COCO()

    # coco_dt = coco_gt.loadRes(coco_results)
    coco_eval = COCOeval(coco_gt, coco_dt, iou_type)
    coco_eval.evaluate()
    coco_eval.accumulate()
    coco_eval.summarize()
    return coco_eval 

def evaluate_predictions_on_coco(
    coco_gt, coco_results, json_result_file, iou_type="bbox"
):
    import json

    with open(json_result_file, "w") as f:
        json.dump(coco_results, f)

    from pycocotools.coco import COCO
    from pycocotools.cocoeval import COCOeval

    coco_dt = coco_gt.loadRes(str(json_result_file)) if coco_results else COCO()

    # coco_dt = coco_gt.loadRes(coco_results)
    coco_eval = COCOeval(coco_gt, coco_dt, iou_type)
    coco_eval.evaluate()
    coco_eval.accumulate()
    coco_eval.summarize()
    return coco_eval 

def finalize_saving_epoch_measures(self):
    self.detections_file.write("]")
    self.detections_file.close()
    self.detections_file = None

    cocoDt = self.coco.loadRes(self.det_file_path)
    from pycocotools.cocoeval import COCOeval
    cocoEval = COCOeval(self.coco, cocoDt, 'bbox')
    cocoEval.evaluate()
    cocoEval.accumulate()
    cocoEval.summarize()
    new_measures = {Measures.MAP_BBOX: cocoEval.stats[0]}

    if self.add_masks:
      cocoEval = COCOeval(self.coco, cocoDt, 'segm')
      cocoEval.evaluate()
      cocoEval.accumulate()
      cocoEval.summarize()
      new_measures[Measures.MAP_SEGM] = cocoEval.stats[0]

    return new_measures 

def evaluate(self, runner, results):
        tmp_file = osp.join(runner.work_dir, 'temp_0')
        result_files = results2json(self.dataset, results, tmp_file)

        res_types = ['bbox', 'segm'
                     ] if runner.model.module.with_mask else ['bbox']
        cocoGt = self.dataset.coco
        imgIds = cocoGt.getImgIds()
        for res_type in res_types:
            cocoDt = cocoGt.loadRes(result_files[res_type])
            iou_type = res_type
            cocoEval = COCOeval(cocoGt, cocoDt, iou_type)
            cocoEval.params.imgIds = imgIds
            cocoEval.evaluate()
            cocoEval.accumulate()
            cocoEval.summarize()
            metrics = ['mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l']
            for i in range(len(metrics)):
                key = '{}_{}'.format(res_type, metrics[i])
                val = float('{:.3f}'.format(cocoEval.stats[i]))
                runner.log_buffer.output[key] = val
            runner.log_buffer.output['{}_mAP_copypaste'.format(res_type)] = (
                '{ap[0]:.3f} {ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} '
                '{ap[4]:.3f} {ap[5]:.3f}').format(ap=cocoEval.stats[:6])
        runner.log_buffer.ready = True
        for res_type in res_types:
            os.remove(result_files[res_type]) 

def _do_python_eval(self, _coco):
        coco_dt = _coco.loadRes(self._result_file)
        coco_eval = COCOeval(_coco, coco_dt)
        coco_eval.params.useSegm = False
        coco_eval.evaluate()
        coco_eval.accumulate()
        self._print_detection_metrics(coco_eval) 

def _update(self, annType='bbox'):
        """Use coco to get real scores. """
        if self._use_ext:
            self.dataset.coco.createIndex(use_ext=True)
            if annType == 'bbox':
                pred = self.dataset.coco.loadRes(self._bbox_filename, use_ext=True)
            else:
                pred = self.dataset.coco.loadRes(self._segm_filename, use_ext=True)
        else:
            if self._results is not None and not self._dump_to_file:
                pred = self.dataset.coco.loadRes(self._results)
            else:
                pred = self.dataset.coco.loadRes(self._filename)
        gt = self.dataset.coco
        # lazy import pycocotools
        from ...data.mscoco.utils import try_import_pycocotools
        try_import_pycocotools()
        from pycocotools.cocoeval import COCOeval
        # only NVIDIA MSCOCO API support use_ext
        coco_eval = COCOeval(gt, pred, annType, use_ext=self._use_ext) \
            if self._use_ext else COCOeval(gt, pred, annType)
        coco_eval.evaluate()
        coco_eval.accumulate()
        names, values = [], []
        names.append('~~~~ Summary {} metrics ~~~~\n'.format(annType))
        # catch coco print string, don't want directly print here
        _stdout = sys.stdout
        sys.stdout = io.StringIO()
        coco_eval.summarize()
        coco_summary = sys.stdout.getvalue()
        sys.stdout = _stdout
        values.append(str(coco_summary).strip())
        names.append('~~~~ Mean AP for {} ~~~~\n'.format(annType))
        values.append('{:.1f}'.format(100 * self._get_ap(coco_eval)))
        return names, values 

def _do_detection_eval(self, res_file, output_dir):
    ann_type = 'bbox'
    coco_dt = self._COCO.loadRes(res_file)
    coco_eval = COCOeval(self._COCO, coco_dt)
    coco_eval.params.useSegm = (ann_type == 'segm')
    coco_eval.evaluate()
    coco_eval.accumulate()
    self._print_detection_eval_metrics(coco_eval)
    eval_file = osp.join(output_dir, 'detection_results.pkl')
    with open(eval_file, 'wb') as fid:
      pickle.dump(coco_eval, fid, pickle.HIGHEST_PROTOCOL)
    print('Wrote COCO eval results to: {}'.format(eval_file)) 

def _do_keypoint_eval(json_dataset, res_file, output_dir):
    ann_type = 'keypoints'
    imgIds = json_dataset.COCO.getImgIds()
    imgIds.sort()
    coco_dt = json_dataset.COCO.loadRes(res_file)
    coco_eval = COCOeval(json_dataset.COCO, coco_dt, ann_type)
    coco_eval.params.imgIds = imgIds
    coco_eval.evaluate()
    coco_eval.accumulate()
    eval_file = os.path.join(output_dir, 'keypoint_results.pkl')
    save_object(coco_eval, eval_file)
    logger.info('Wrote json eval results to: {}'.format(eval_file))
    coco_eval.summarize()
    return coco_eval 

def compute_ap(self):
        coco_res = self.loader.coco.loadRes(self.filename)

        cocoEval = COCOeval(self.loader.coco, coco_res)
        cocoEval.params.imgIds = self.loader.get_filenames()
        cocoEval.params.useSegm = False

        cocoEval.evaluate()
        cocoEval.accumulate()
        cocoEval.summarize()
        return cocoEval 

def coco_eval(result_files, result_types, coco, max_dets=(100, 300, 1000)):
    for res_type in result_types:
        assert res_type in [
            'proposal', 'proposal_fast', 'bbox', 'segm', 'keypoints'
        ]

    if mmcv.is_str(coco):
        coco = COCO(coco)
    assert isinstance(coco, COCO)

    if result_types == ['proposal_fast']:
        ar = fast_eval_recall(result_files, coco, np.array(max_dets))
        for i, num in enumerate(max_dets):
            print('AR@{}\t= {:.4f}'.format(num, ar[i]))
        return

    for res_type in result_types:
        result_file = result_files[res_type]
        assert result_file.endswith('.json')

        coco_dets = coco.loadRes(result_file)
        img_ids = coco.getImgIds()
        iou_type = 'bbox' if res_type == 'proposal' else res_type
        cocoEval = COCOeval(coco, coco_dets, iou_type)
        cocoEval.params.imgIds = img_ids
        if res_type == 'proposal':
            cocoEval.params.useCats = 0
            cocoEval.params.maxDets = list(max_dets)
        cocoEval.evaluate()
        cocoEval.accumulate()
        cocoEval.summarize() 

def cocoval(self, detected_json, eval_json):
    eval_gt = COCO(eval_json)
    eval_dt = eval_gt.loadRes(detected_json)
    cocoEval = COCOeval(eval_gt, eval_dt, iouType='bbox')

    cocoEval.evaluate()
    cocoEval.accumulate()
    cocoEval.summarize() 

def _update(self):
        """Use coco to get real scores. """
        if not self._current_id == len(self._img_ids):
            warnings.warn(
                'Recorded {} out of {} validation images, incomplete results'.format(
                    self._current_id, len(self._img_ids)))
        if not self._results:
            # in case of empty results, push a dummy result
            self._results.append({'image_id': self._img_ids[0],
                                  'category_id': 0,
                                  'bbox': [0, 0, 0, 0],
                                  'score': 0})
        import json
        try:
            with open(self._filename, 'w') as f:
                json.dump(self._results, f)
        except IOError as e:
            raise RuntimeError("Unable to dump json file, ignored. What(): {}".format(str(e)))

        pred = self.dataset.coco.loadRes(self._filename)
        gt = self.dataset.coco
        # lazy import pycocotools
        from ...data.mscoco.utils import try_import_pycocotools
        try_import_pycocotools()
        from pycocotools.cocoeval import COCOeval
        coco_eval = COCOeval(gt, pred, 'bbox')
        coco_eval.evaluate()
        coco_eval.accumulate()
        self._coco_eval = coco_eval
        return coco_eval 

def cocoval(self, detected_json, eval_json):
    eval_gt = COCO(eval_json)
    eval_dt = eval_gt.loadRes(detected_json)
    cocoEval = COCOeval(eval_gt, eval_dt, iouType='bbox')

    cocoEval.evaluate()
    cocoEval.accumulate()
    cocoEval.summarize()