Python motmetrics.MOTAccumulator() Examples

def __init__(self, auto_id=False, max_switch_time=float('inf')):
        """Create a MOTAccumulator.

        Params
        ------
        auto_id : bool, optional
            Whether or not frame indices are auto-incremented or provided upon
            updating. Defaults to false. Not specifying a frame-id when this value
            is true results in an error. Specifying a frame-id when this value is
            false also results in an error.

        max_switch_time : scalar, optional
            Allows specifying an upper bound on the timespan an unobserved but 
            tracked object is allowed to generate track switch events. Useful if groundtruth 
            objects leaving the field of view keep their ID when they reappear, 
            but your tracker is not capable of recognizing this (resulting in 
            track switch events). The default is that there is no upper bound
            on the timespan. In units of frame timestamps. When using auto_id
            in units of count.
        """

        self.auto_id = auto_id
        self.max_switch_time = max_switch_time
        self.reset() 

def __init__(self, obstacle_tracking_stream, ground_obstacles_stream,
                 flags):
        obstacle_tracking_stream.add_callback(self.on_tracker_obstacles)
        ground_obstacles_stream.add_callback(self.on_ground_obstacles)
        erdos.add_watermark_callback(
            [obstacle_tracking_stream, ground_obstacles_stream], [],
            self.on_watermark)
        self._flags = flags
        self._logger = erdos.utils.setup_logging(self.config.name,
                                                 self.config.log_file_name)
        self._csv_logger = erdos.utils.setup_csv_logging(
            self.config.name + '-csv', self.config.csv_log_file_name)
        self._last_notification = None
        # Buffer of detected obstacles.
        self._tracked_obstacles = []
        # Buffer of ground obstacles.
        self._ground_obstacles = []
        # Heap storing pairs of (ground/output time, game time).
        self._tracker_start_end_times = []
        self._sim_interval = None
        self._accumulator = mm.MOTAccumulator(auto_id=True)
        self._metrics_host = mm.metrics.create() 

def __init__(self, auto_id=False, max_switch_time=float('inf')):
        """Create a MOTAccumulator.

        Params
        ------
        auto_id : bool, optional
            Whether or not frame indices are auto-incremented or provided upon
            updating. Defaults to false. Not specifying a frame-id when this value
            is true results in an error. Specifying a frame-id when this value is
            false also results in an error.

        max_switch_time : scalar, optional
            Allows specifying an upper bound on the timespan an unobserved but 
            tracked object is allowed to generate track switch events. Useful if groundtruth 
            objects leaving the field of view keep their ID when they reappear, 
            but your tracker is not capable of recognizing this (resulting in 
            track switch events). The default is that there is no upper bound
            on the timespan. In units of frame timestamps. When using auto_id
            in units of count.
        """

        self.auto_id = auto_id
        self.max_switch_time = max_switch_time
        self.reset() 

def test_correct_average():
    # Tests what is being depicted in figure 3 of 'Evaluating MOT Performance'
    acc = mm.MOTAccumulator(auto_id=True)

    # No track
    acc.update([1, 2, 3, 4], [], [])
    acc.update([1, 2, 3, 4], [], [])
    acc.update([1, 2, 3, 4], [], [])
    acc.update([1, 2, 3, 4], [], [])

    # Track single
    acc.update([4], [4], [0])
    acc.update([4], [4], [0])
    acc.update([4], [4], [0])
    acc.update([4], [4], [0])

    mh = mm.metrics.create()
    metr = mh.compute(acc, metrics='mota', return_dataframe=False)
    assert metr['mota'] == approx(0.2) 

def reset(self):
        """Reset the accumulator to empty state."""

        self.events = MOTAccumulator.new_event_dataframe()
        self.m = {} # Pairings up to current timestamp  
        self.last_occurrence = {} # Tracks most recent occurance of object 

def test_events():
    acc = mm.MOTAccumulator()

    # All FP
    acc.update([], ['a', 'b'], [], frameid=0)
    # All miss
    acc.update([1, 2], [], [], frameid=1)
    # Match
    acc.update([1, 2], ['a', 'b'], [[1, 0.5], [0.3, 1]], frameid=2)
    # Switch
    acc.update([1, 2], ['a', 'b'], [[0.2, np.nan], [np.nan, 0.1]], frameid=3)
    # Match. Better new match is available but should prefer history
    acc.update([1, 2], ['a', 'b'], [[5, 1], [1, 5]], frameid=4)
    # No data
    acc.update([], [], [], frameid=5)

    expect = mm.MOTAccumulator.new_event_dataframe()
    expect.loc[(0, 0), :] = ['FP', np.nan, 'a', np.nan]
    expect.loc[(0, 1), :] = ['FP', np.nan, 'b', np.nan]
    expect.loc[(1, 0), :] = ['MISS', 1, np.nan, np.nan]
    expect.loc[(1, 1), :] = ['MISS', 2, np.nan, np.nan]
    expect.loc[(2, 0), :] = ['MATCH', 1, 'b', 0.5]
    expect.loc[(2, 1), :] = ['MATCH', 2, 'a', 0.3]
    expect.loc[(3, 0), :] = ['SWITCH', 1, 'a', 0.2]
    expect.loc[(3, 1), :] = ['SWITCH', 2, 'b', 0.1]
    expect.loc[(4, 0), :] = ['MATCH', 1, 'a', 5.]
    expect.loc[(4, 1), :] = ['MATCH', 2, 'b', 5.]
    # frame 5 generates no events

    assert pd.DataFrame.equals(acc.events, expect)

    mh = mm.metrics.create()
    metr = mh.compute(acc, metrics=['motp', 'mota'], return_dataframe=False)
    #print(metr)
    #print(approx((1.0 - (((2 + 2) + 2) / 8))))
    assert metr['motp'] == approx(11.1 / 6)
    assert metr['mota'] == approx(1. - (2 + 2 + 2) / 8.) 

def test_auto_id():
    acc = mm.MOTAccumulator(auto_id=True)
    acc.update([1, 2, 3, 4], [], [])
    acc.update([1, 2, 3, 4], [], [])
    assert acc.events.index.levels[0][-1] == 1
    acc.update([1, 2, 3, 4], [], [])
    assert acc.events.index.levels[0][-1] == 2

    with pytest.raises(AssertionError):
        acc.update([1, 2, 3, 4], [], [], frameid=5)

    acc = mm.MOTAccumulator(auto_id=False)
    with pytest.raises(AssertionError):
        acc.update([1, 2, 3, 4], [], []) 

def reset_accumulator(self):
        self.acc = mm.MOTAccumulator(auto_id=True) 

def reset_accumulator(self):
        self.acc = mm.MOTAccumulator(auto_id=True) 

def reset(self):
        """Reset the accumulator to empty state."""

        self.events = MOTAccumulator.new_event_dataframe()
        self.m = {} # Pairings up to current timestamp  
        self.last_occurrence = {} # Tracks most recent occurance of object 

def test_events():
    acc = mm.MOTAccumulator()

    # All FP
    acc.update([], ['a', 'b'], [], frameid=0)
    # All miss
    acc.update([1, 2], [], [], frameid=1)
    # Match
    acc.update([1, 2], ['a', 'b'], [[1, 0.5], [0.3, 1]], frameid=2)
    # Switch
    acc.update([1, 2], ['a', 'b'], [[0.2, np.nan], [np.nan, 0.1]], frameid=3)
    # Match. Better new match is available but should prefer history
    acc.update([1, 2], ['a', 'b'], [[5, 1], [1, 5]], frameid=4)
    # No data
    acc.update([], [], [], frameid=5)

    expect = mm.MOTAccumulator.new_event_dataframe()
    expect.loc[(0, 0), :] = ['FP', np.nan, 'a', np.nan]
    expect.loc[(0, 1), :] = ['FP', np.nan, 'b', np.nan]
    expect.loc[(1, 0), :] = ['MISS', 1, np.nan, np.nan]
    expect.loc[(1, 1), :] = ['MISS', 2, np.nan, np.nan]
    expect.loc[(2, 0), :] = ['MATCH', 1, 'b', 0.5]
    expect.loc[(2, 1), :] = ['MATCH', 2, 'a', 0.3]
    expect.loc[(3, 0), :] = ['SWITCH', 1, 'a', 0.2]
    expect.loc[(3, 1), :] = ['SWITCH', 2, 'b', 0.1]
    expect.loc[(4, 0), :] = ['MATCH', 1, 'a', 5.]
    expect.loc[(4, 1), :] = ['MATCH', 2, 'b', 5.]
    # frame 5 generates no events

    assert pd.DataFrame.equals(acc.events, expect)
    
    mh = mm.metrics.create()
    metr = mh.compute(acc, metrics=['motp', 'mota'], return_dataframe=False)
    assert metr['motp'] == approx(11.1 / 6)
    assert metr['mota'] == approx(1. - (2 + 2 + 2) / 8) 

def test_max_switch_time():
     acc = mm.MOTAccumulator(max_switch_time=1)
     df = acc.update([1, 2], ['a', 'b'], [[1, 0.5], [0.3, 1]], frameid=1) # 1->a, 2->b
     df = acc.update([1, 2], ['a', 'b'], [[0.5, np.nan], [np.nan, 0.5]], frameid=2) # 1->b, 2->a 
     assert (df.Type == 'SWITCH').all()

     acc = mm.MOTAccumulator(max_switch_time=1)
     df = acc.update([1, 2], ['a', 'b'], [[1, 0.5], [0.3, 1]], frameid=1) # 1->a, 2->b
     df = acc.update([1, 2], ['a', 'b'], [[0.5, np.nan], [np.nan, 0.5]], frameid=5) # Later frame 1->b, 2->a 
     assert (df.Type == 'MATCH').all() 

def reset_accumulator(self):
        self.acc = mm.MOTAccumulator(auto_id=True) 

def test_max_switch_time():
     acc = mm.MOTAccumulator(max_switch_time=1)
     df = acc.update([1, 2], ['a', 'b'], [[1, 0.5], [0.3, 1]], frameid=1) # 1->a, 2->b
     df = acc.update([1, 2], ['a', 'b'], [[0.5, np.nan], [np.nan, 0.5]], frameid=2) # 1->b, 2->a
     assert (df.Type == 'SWITCH').all()

     acc = mm.MOTAccumulator(max_switch_time=1)
     df = acc.update([1, 2], ['a', 'b'], [[1, 0.5], [0.3, 1]], frameid=1) # 1->a, 2->b
     df = acc.update([1, 2], ['a', 'b'], [[0.5, np.nan], [np.nan, 0.5]], frameid=5) # Later frame 1->b, 2->a
     assert (df.Type == 'MATCH').all() 

def get_mot_accum(results, seq):
    mot_accum = mm.MOTAccumulator(auto_id=True)    

    for i, data in enumerate(seq):
        gt = data['gt']
        gt_ids = []
        if gt:
            gt_boxes = []
            for gt_id, box in gt.items():
                gt_ids.append(gt_id)
                gt_boxes.append(box)
        
            gt_boxes = np.stack(gt_boxes, axis=0)
            # x1, y1, x2, y2 --> x1, y1, width, height
            gt_boxes = np.stack((gt_boxes[:, 0],
                                 gt_boxes[:, 1],
                                 gt_boxes[:, 2] - gt_boxes[:, 0],
                                 gt_boxes[:, 3] - gt_boxes[:, 1]),
                                axis=1)
        else:
            gt_boxes = np.array([])
        
        track_ids = []
        track_boxes = []
        for track_id, frames in results.items():
            if i in frames:
                track_ids.append(track_id)
                # frames = x1, y1, x2, y2, score
                track_boxes.append(frames[i][:4])

        if track_ids:
            track_boxes = np.stack(track_boxes, axis=0)
            # x1, y1, x2, y2 --> x1, y1, width, height
            track_boxes = np.stack((track_boxes[:, 0],
                                    track_boxes[:, 1],
                                    track_boxes[:, 2] - track_boxes[:, 0],
                                    track_boxes[:, 3] - track_boxes[:, 1]),
                                    axis=1)
        else:
            track_boxes = np.array([])
        
        distance = mm.distances.iou_matrix(gt_boxes, track_boxes, max_iou=0.5)
        
        mot_accum.update(
            gt_ids,
            track_ids,
            distance)

    return mot_accum 

def process_frame(si, nJoints, seqidxs, seqidxsUniq, motAll, metricsMidNames):
    # create metrics
    mh = mm.metrics.create()
    print("seqidx: %d" % (si+1))

    # init per-joint metrics accumulator
    accAll = {}
    for i in range(nJoints):
        accAll[i] = mm.MOTAccumulator(auto_id=True)

    # extract frames IDs for the sequence
    imgidxs = np.argwhere(seqidxs == seqidxsUniq[si])
    # DEBUG: remove the last frame of each sequence from evaluation due to buggy annotations
    print("DEBUG: remove last frame from eval until annotations are fixed")
    imgidxs = imgidxs[:-1].copy()
    # create an accumulator that will be updated during each frame
    # iterate over frames
    for j in range(len(imgidxs)):
        imgidx = imgidxs[j,0]
        # iterate over joints
        for i in range(nJoints):
            # GT tracking ID
            trackidxGT = motAll[imgidx][i]["trackidxGT"]
            # prediction tracking ID
            trackidxPr = motAll[imgidx][i]["trackidxPr"]
            # distance GT <-> pred part to compute MOT metrics
            # 'NaN' means force no match
            dist = motAll[imgidx][i]["dist"]
            # Call update once per frame
            accAll[i].update(
                trackidxGT,  # Ground truth objects in this frame
                trackidxPr,  # Detector hypotheses in this frame
                dist  # Distances from objects to hypotheses
            )

    # compute intermediate metrics per joint per sequence
    all_metricsMid = []
    for i in range(nJoints):
        all_metricsMid.append(
            mh.compute(accAll[i], metrics=metricsMidNames,
                       return_dataframe=False, name='acc'))
    return all_metricsMid, accAll