Python cv2.mean() Examples

def add_median_colr(img_ori, img_mask, mask, min_bbox):
    img_crop = faceCrop(img_ori, min_bbox, scale_ratio=0.5)
    (B, G, R) = cv2.split(img_crop)
    B_median = np.median(B)
    G_median = np.median(G)
    R_median = np.median(R)
    # mean_pixel = cv2.mean(img[int(rect[1]):int(rect[3]), int(rect[0]):int(rect[2])])  # get img mean pixel
    rows, cols, _ = img_ori.shape
    for row in range(rows):
        for col in range(cols):
            if mask[row, col] < 1:
                img_mask[row, col][0] = B_median
                img_mask[row, col][1] = G_median
                img_mask[row, col][2] = R_median

    return img_mask 

def roiColor(self, image):
        """
        Finds the averaged color within the ROI within the square. The ROI is a circle with radius r from
        the centre of the square.
        """
        # Initialise mask
        maskImage = np.zeros((image.shape[0], image.shape[1]), np.uint8)
        # Draw the ROI circle on the mask
        cv2.circle(maskImage, self.roi, self.radius, (255, 255, 255), -1)
        # Find the average color
        average_raw = cv2.mean(image, mask=maskImage)[::-1]
        # Need int format so reassign variable
        average = (int(average_raw[1]), int(average_raw[2]), int(average_raw[3]))

        ## DEBUG
        # print(average)

        return average 

def __init__(self, max_half_life=500. * 1000, min_half_life=5.* 1000, increment = 1.2):
        # the maximal half life of a pixel from background, in seconds
        self._max_half_life = float(max_half_life)
        # the minimal one
        self._min_half_life = float(min_half_life)

        # starts with the fastest learning rate
        self._current_half_life = self._min_half_life

        # fixme theoretically this should depend on time, not frame index
        self._increment = increment
        # the mean background
        self._bg_mean = None
        # self._bg_sd = None

        self._buff_alpha_matrix = None
        self._buff_invert_alpha_mat = None
        # the time stamp of the frame las used to update
        self.last_t = 0 

def calculate_color_diff(image, mask_1, mask_2):
        mean_1 = cv2.mean(image, mask_1)[:3]
        color_1 = Color(mean_1, scaling=Scaling.UINT8, space=Space.BGR)
        mean_2 = cv2.mean(image, mask_2)[:3]
        color_2 = Color(mean_2, scaling=Scaling.UINT8, space=Space.BGR)

        color_diff = sqrt(sum([(c1 - c2) ** 2 for c1, c2 in zip(color_1(), color_2())]))

        return color_diff 

def anonymize_face_pixelate(image, blocks=3):
    # divide the input image into NxN blocks
    (h, w) = image.shape[:2]
    xSteps = np.linspace(0, w, blocks + 1, dtype="int")
    ySteps = np.linspace(0, h, blocks + 1, dtype="int")
    # loop over the blocks in both the x and y direction
    for i in range(1, len(ySteps)):
        for j in range(1, len(xSteps)):
            # compute the starting and ending (x, y)-coordinates
            # for the current block
            startX = xSteps[j - 1]
            startY = ySteps[i - 1]
            endX = xSteps[j]
            endY = ySteps[i]
            # extract the ROI using NumPy array slicing, compute the
            # mean of the ROI, and then draw a rectangle with the
            # mean RGB values over the ROI in the original image
            roi = image[startY:endY, startX:endX]
            (B, G, R) = [int(x) for x in cv2.mean(roi)[:3]]
            cv2.rectangle(image, (startX, startY), (endX, endY), (B, G, R), -1)
    # return the pixelated blurred image
    return image


# Filters path 

def shade(self, polygons: np.ndarray, image: np.ndarray) -> np.ndarray:
        canvas_dimensions = self.get_output_dimensions(image)
        scale_factor = max(canvas_dimensions) / max(image.shape)
        scaled_polygons = polygons * scale_factor
        output_image = np.zeros(canvas_dimensions, dtype=np.uint8)
        for polygon, scaled_polygon in zip(polygons, scaled_polygons):
            polygon = self.strip_negative_points(polygon)
            scaled_polygon = self.strip_negative_points(scaled_polygon)
            if len(polygon) < 3:
                continue
            mask = np.zeros(image.shape[:2], dtype=np.uint8)
            cv2.fillConvexPoly(mask, polygon, (255,))
            color = self.get_dominant_color(image[mask > 0], 3, 3).tolist()
            # color = cv2.mean(image, mask)[:3]
            cv2.fillConvexPoly(output_image, scaled_polygon.astype(np.int32), color, lineType=cv2.LINE_AA)
        return output_image 

def retrieve_area_color(data, contour, labels):
    """Mask an image area and retrieve its dominant color starting from a label
    glossary, by determining its closest label (regarding euclidean distance).

    Largely inspired from : https://www.pyimagesearch.com/\
    2016/02/15/determining-object-color-with-opencv/

    Parameters
    ----------
    data : np.array
        3-channelled image
    contour : np.array
        List of points that delimits the area
    labels : list
        List of dictionnary that describes each labels (with "id" and "color" keys)
    """
    mask = np.zeros(data.shape[:2], dtype="uint8")
    cv2.drawContours(mask, [contour], -1, 255, -1)
    mean = cv2.mean(data, mask=mask)[:3]
    min_dist = (np.inf, None)
    for label in labels:
        d = np.linalg.norm(label["color"] - np.array(mean))
        if d < min_dist[0]:
            min_dist = (d, label["id"])
    return min_dist[1] 

def detect_single_scale(score_map, geo_map, score_map_thresh, nms_thres, box_thresh, timer):
    if len(score_map.shape) == 4:
        score_map = score_map[0, :, :, 0]
        geo_map = geo_map[0, :, :, ]
    # filter the score map
    xy_text = np.argwhere(score_map > score_map_thresh)
    # sort the text boxes via the y axis
    xy_text = xy_text[np.argsort(xy_text[:, 0])]
    # restore
    start = time.time()
    # xy_text[:, ::-1]*4 满足条件的pixel的坐标
    # geo_map[xy_text[:, 0], xy_text[:, 1], :] 得到对应点到ｂｏｕｎｄｉｎｇ　ｂｏｘ　的距离
    text_box_restored = restore_rectangle(xy_text[:, ::-1], geo_map[xy_text[:, 0], xy_text[:, 1], :])  # N*4*2
    print('{} text boxes before nms'.format(text_box_restored.shape[0]))
    boxes = np.zeros((text_box_restored.shape[0], 9), dtype=np.float32)
    boxes[:, :8] = text_box_restored.reshape((-1, 8))
    boxes[:, 8] = score_map[xy_text[:, 0], xy_text[:, 1]]
    timer['restore'] = time.time() - start
    # nms part
    start = time.time()
    # boxes = nms_locality.nms_locality(boxes.astype(np.float64), nms_thres)
    boxes = lanms.merge_quadrangle_n9(boxes.astype('float32'), nms_thres)
    timer['nms'] = time.time() - start

    if boxes.shape[0] == 0:
        return None, timer

    # here we filter some low score boxes by the average score map, this is different from the orginal paper
    for i, box in enumerate(boxes):
        mask = np.zeros_like(score_map, dtype=np.uint8)
        cv2.fillPoly(mask, box[:8].reshape((-1, 4, 2)).astype(np.int32), 1)
        boxes[i, 8] = cv2.mean(score_map, mask)[0]
    boxes = boxes[boxes[:, 8] > box_thresh]
    return boxes 

def process(self, raw, identifiers=None, frame_meta=None):
        raw_outputs = self._extract_predictions(raw, frame_meta)
        score_maps = raw_outputs[self.score_map_out]
        geometry_maps = raw_outputs[self.geometry_map_out]
        is_nchw = score_maps.shape[1] == 1
        results = []
        if is_nchw:
            score_maps = np.transpose(score_maps, (0, 2, 3, 1))
            geometry_maps = np.transpose(geometry_maps, (0, 2, 3, 1))
        for identifier, score_map, geo_map, meta in zip(identifiers, score_maps, geometry_maps, frame_meta):
            if len(score_map.shape) == 3:
                score_map = score_map[:, :, 0]
                geo_map = geo_map[:, :, ]
            xy_text = np.argwhere(score_map > self.score_map_thresh)
            xy_text = xy_text[np.argsort(xy_text[:, 0])]
            text_box_restored = self.restore_rectangle(
                xy_text[:, ::-1] * 4, geo_map[xy_text[:, 0], xy_text[:, 1], :]
            )  # N*4*2
            boxes = np.zeros((text_box_restored.shape[0], 9), dtype=np.float32)
            boxes[:, :8] = text_box_restored.reshape((-1, 8))
            boxes[:, 8] = score_map[xy_text[:, 0], xy_text[:, 1]]

            boxes = self.nms_locality(boxes.astype('float32'), self.nms_thresh)

            if boxes.shape[0] == 0:
                results.append(TextDetectionPrediction(identifier, boxes))
                continue
            for i, box in enumerate(boxes):
                mask = np.zeros_like(score_map, dtype=np.uint8)
                cv2.fillPoly(mask, box[:8].reshape((-1, 4, 2)).astype(np.int32) // 4, 1)
                boxes[i, 8] = cv2.mean(score_map, mask)[0]
            boxes = boxes[boxes[:, 8] > self.box_thresh]

            boxes = boxes[:, :8].reshape((-1, 4, 2))
            boxes[:, :, 0] /= meta.get('scale_x', 1)
            boxes[:, :, 1] /= meta.get('scale_y', 1)
            results.append(TextDetectionPrediction(identifier, boxes))

        return results 

def imgproc(frame):
    
    # convert color to gray scale and show it
    
    background = np.full_like(frame, 0)
    background[:,:,1] = 162
    background[:,:,2] = 14
    
    
    img = frame - background
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    
    # convert image to black and white and show it
    thresh1, thresh = cv2.threshold(gray, 10, 255, cv2.THRESH_BINARY)
    
    # find contours!
    contours, hry = cv2.findContours(thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    
    # draw all the contours
    cpframe = frame.copy()
    cv2.drawContours(cpframe, contours, -1, (255,0,0), 3)
    cv2.imshow('contours', cpframe)
    
    # ================== TODO ===================
    
    # Every pill is surrounded by a contour in variable "contours" now
    
    # ============================================
    
    for ctr in contours:
        m = cv2.mean(ctr)
        org = int(m[0]), int(m[1])
        cv2.putText(frame, "pill", org, cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 0, 0), 2)
    
    return frame 

def box_score_fast(self, bitmap, _box):
        h, w = bitmap.shape[:2]
        box = _box.copy()
        xmin = np.clip(np.floor(box[:, 0].min()).astype(np.int), 0, w - 1)
        xmax = np.clip(np.ceil(box[:, 0].max()).astype(np.int), 0, w - 1)
        ymin = np.clip(np.floor(box[:, 1].min()).astype(np.int), 0, h - 1)
        ymax = np.clip(np.ceil(box[:, 1].max()).astype(np.int), 0, h - 1)

        mask = np.zeros((ymax - ymin + 1, xmax - xmin + 1), dtype=np.uint8)
        box[:, 0] = box[:, 0] - xmin
        box[:, 1] = box[:, 1] - ymin
        cv2.fillPoly(mask, box.reshape(1, -1, 2).astype(np.int32), 1)
        return cv2.mean(bitmap[ymin:ymax + 1, xmin:xmax + 1], mask)[0] 

def transform_for_test():
    """
    CV2 => PI => tensor
    """
    #image = Image.fromarray(np.uint8(img))

    transform_list = []
    
    transform_list.append(transforms.ToTensor())
    
    transform_list.append(transforms.Normalize(mean=(0.5,0.5,0.5),std=(0.5,0.5,0.5)))

    transform = transforms.Compose(transform_list)

    return transform 

def detect(score_map, geo_map, timer, score_map_thresh=1e-5, box_thresh=1e-8, nms_thres=0.1):
    '''
    restore text boxes from score map and geo map
    :param score_map:
    :param geo_map:
    :param timer:
    :param score_map_thresh: threshhold for score map
    :param box_thresh: threshhold for boxes
    :param nms_thres: threshold for nms
    :return:
    '''
    if len(score_map.shape) == 4:
        score_map = score_map[0, :, :, 0]
        geo_map = geo_map[0, :, :, ]
    # filter the score map
    xy_text = np.argwhere(score_map > score_map_thresh)
    # sort the text boxes via the y axis
    xy_text = xy_text[np.argsort(xy_text[:, 0])]
    # restore
    start = time.time()
    text_box_restored = restore_rectangle(xy_text[:, ::-1]*4, geo_map[xy_text[:, 0], xy_text[:, 1], :]) # N*4*2
    #print('{} text boxes before nms'.format(text_box_restored.shape[0]))
    boxes = np.zeros((text_box_restored.shape[0], 9), dtype=np.float32)
    boxes[:, :8] = text_box_restored.reshape((-1, 8))
    boxes[:, 8] = score_map[xy_text[:, 0], xy_text[:, 1]]
    timer['restore'] = time.time() - start
    # nms part
    start = time.time()
    # boxes = nms_locality.nms_locality(boxes.astype(np.float64), nms_thres)
    boxes = lanms.merge_quadrangle_n9(boxes.astype('float32'), nms_thres)
    timer['nms'] = time.time() - start
    if boxes.shape[0] == 0:
        return None, timer

    # here we filter some low score boxes by the average score map, this is different from the orginal paper
    for i, box in enumerate(boxes):
        mask = np.zeros_like(score_map, dtype=np.uint8)
        cv2.fillPoly(mask, box[:8].reshape((-1, 4, 2)).astype(np.int32) // 4, 1)
        boxes[i, 8] = cv2.mean(score_map, mask)[0]
    boxes = boxes[boxes[:, 8] > box_thresh]
    return boxes, timer 

def isRaidSighting(self, img):
        ret = True
        LOG.debug('image mean :{}'.format(img.mean()))
        if int(img.mean()) > 240:
            LOG.debug('No raid sightings')
            ret = False
        return ret 

def detectEgg(self, time_img):
        img_gray = cv2.cvtColor(time_img, cv2.COLOR_BGR2GRAY)
        ret, thresh1 = cv2.threshold(img_gray, 220, 255, cv2.THRESH_BINARY_INV)
        kernel = np.ones((2, 2), np.uint8)
        thresh1 = cv2.erode(thresh1, kernel, iterations=1)
        time_mean = cv2.mean(time_img, thresh1)
        if time_mean[2] > (time_mean[0]+50): # Red is greater than Blue+50
            return False, thresh1
        else:
            return True, thresh1 

def detectRaidBossTimer(self, time_img, scale):
        text = ''
        if int(time_img.mean()) > 240:
            return text
        time_img = cv2.resize(time_img, None, fx=1.0/scale, fy=1.0/scale, interpolation=cv2.INTER_CUBIC)
        cv2.imwrite(self.timefile,time_img)
        text = pytesseract.image_to_string(Image.open(self.timefile),config='-c tessedit_char_whitelist=1234567890: -psm 7')
        return text 

def detect(score_map, geo_map, score_map_thresh=0.8, box_thresh=0.1, nms_thres=0.2):                                                
    '''                                                                                                                                    
    restore text boxes from score map and geo map
    :param score_map: 
    :param geo_map:
    :param score_map_thresh: threshhold for score map
    :param box_thresh: threshhold for boxes
    :param nms_thres: threshold for nms
    :return:
    '''

    if len(score_map.shape) == 4:
        score_map = score_map[0, :, :, 0]
        geo_map = geo_map[0, :, :, ]

    # filter the score map
    xy_text = np.argwhere(score_map > score_map_thresh)
    # sort the text boxes via the y axis                                                                                                   
    xy_text = xy_text[np.argsort(xy_text[:, 0])]
    # restore                                                                                                                              

    text_box_restored = restore_rectangle(xy_text[:, ::-1]*4, geo_map[xy_text[:, 0], xy_text[:, 1], :]) # N*4*2
    print('{} text boxes before nms'.format(text_box_restored.shape[0]))
    boxes = np.zeros((text_box_restored.shape[0], 9), dtype=np.float32)
    boxes[:, :8] = text_box_restored.reshape((-1, 8))
    boxes[:, 8] = score_map[xy_text[:, 0], xy_text[:, 1]]
    # boxes = nms_locality.nms_locality(boxes.astype(np.float64), nms_thres)
    boxes = lanms.merge_quadrangle_n9(boxes.astype('float32'), nms_thres)

    if boxes.shape[0] == 0:
        return None

    # here we filter some low score boxes by the average score map, this is different from the orginal paper                               
    for i, box in enumerate(boxes):
        mask = np.zeros_like(score_map, dtype=np.uint8)
        cv2.fillPoly(mask, box[:8].reshape((-1, 4, 2)).astype(np.int32) // 4, 1)
        boxes[i, 8] = cv2.mean(score_map, mask)[0]
    boxes = boxes[boxes[:, 8] > box_thresh]

    return boxes 

def remove_points_with_big_reproj_err(self, points): 
        with self._lock:             
            with self.update_lock: 
                #print('map points: ', sorted([p.id for p in self.points]))
                #print('points: ', sorted([p.id for p in points]))           
                culled_pt_count = 0
                for p in points:
                    # compute reprojection error
                    chi2s = []
                    for f, idx in p.observations():
                        uv = f.kpsu[idx]
                        proj,_ = f.project_map_point(p)
                        invSigma2 = Frame.feature_manager.inv_level_sigmas2[f.octaves[idx]]
                        err = (proj-uv)
                        chi2s.append(np.inner(err,err)*invSigma2)
                    # cull
                    mean_chi2 = np.mean(chi2s)
                    if np.mean(chi2s) > Parameters.kChi2Mono:  # chi-square 2 DOFs  (Hartley Zisserman pg 119)
                        culled_pt_count += 1
                        #print('removing point: ',p.id, 'from frames: ', [f.id for f in p.keyframes])
                        self.remove_point(p)
                Printer.blue("# culled map points: ", culled_pt_count)        


    # BA considering all keyframes: 
    # - local keyframes are adjusted, 
    # - other keyframes are fixed
    # - all points are adjusted 

def detect(self, score_map, geo_map, score_map_thresh=0.8, box_thresh=0.1, nms_thres=0.2):                                                
    '''                                                                                                                                    
    restore text boxes from score map and geo map                                                                                          
    :param score_map:                                                                                                                      
    :param geo_map:                                                                                                                    
    :param score_map_thresh: threshhold for score map                                                                                      
    :param box_thresh: threshhold for boxes                                                                                                
    :param nms_thres: threshold for nms                                                                                                    
    :return:                                                                                                                               
    '''
    if len(score_map.shape) == 4:                                                                                                          
      score_map = score_map[0, :, :, 0]                                                                                                  
      geo_map = geo_map[0, :, :, ]                                                                                                       
    # filter the score map                                                                                                                 
    xy_text = np.argwhere(score_map > score_map_thresh)                                                                                    
    # sort the text boxes via the y axis                                                                                                   
    xy_text = xy_text[np.argsort(xy_text[:, 0])]                                                                                           
    # restore                                                                                                                              

    text_box_restored = restore_rectangle(xy_text[:, ::-1]*4, geo_map[xy_text[:, 0], xy_text[:, 1], :]) # N*4*2                            
    print('{} text boxes before nms'.format(text_box_restored.shape[0]))                                                                   
    boxes = np.zeros((text_box_restored.shape[0], 9), dtype=np.float32)                                                                    
    boxes[:, :8] = text_box_restored.reshape((-1, 8))                                                                                      
    boxes[:, 8] = score_map[xy_text[:, 0], xy_text[:, 1]]                                                                                  
    # boxes = nms_locality.nms_locality(boxes.astype(np.float64), nms_thres)                                                               
    boxes = lanms.merge_quadrangle_n9(boxes.astype('float32'), nms_thres)                                                                                                                                                                   
                                                                                                                                           
    if boxes.shape[0] == 0:                                                                                                                
      return None                                                                                                               
                                                                                                                                           
    # here we filter some low score boxes by the average score map, this is different from the orginal paper                               
    for i, box in enumerate(boxes):                                                                                                        
      mask = np.zeros_like(score_map, dtype=np.uint8)                                                                                    
      cv2.fillPoly(mask, box[:8].reshape((-1, 4, 2)).astype(np.int32) // 4, 1)                                                           
      boxes[i, 8] = cv2.mean(score_map, mask)[0]                                                                                         
    boxes = boxes[boxes[:, 8] > box_thresh]                                                                                                
                                                                                                                                           
    return boxes 

def _comput_blob_features(self, img, contour, lr = 1e-5):
        hull = contour

        x,y,w,h = cv2.boundingRect(contour)

        roi = img[y : y + h, x : x + w]
        mask = np.zeros_like(roi)
        cv2.drawContours(mask,[hull],-1, 1,-1,offset=(-x,-y))

        mean_col = cv2.mean(roi,mask)[0]


        if len(self.positions) > 2:

            last_two_pos = self._positions.tail(2)
            xm, xmm = last_two_pos.x
            ym, ymm = last_two_pos.y

            instantaneous_speed = abs(xm + 1j*ym - xmm + 1j*ymm)
        else:
            instantaneous_speed = 0
        if np.isnan(instantaneous_speed):
            instantaneous_speed = 0

        features = np.array([cv2.contourArea(hull) + 1.0,
                            cv2.arcLength(hull,True) + 1.0,
                            instantaneous_speed +1.0,
                            mean_col +1
                             ])

        return features 

def distance(self, features,time):
        if time - self._last_updated_time > self._max_unupdated_duration:
            logging.warning("FG model not updated for too long. Resetting.")
            self.__init__(self._history_length)
            return 0

        if not self._is_ready:
            last_row = self._ring_buff_idx + 1
        else:
            last_row = self._history_length

        means = np.mean(self._ring_buff[:last_row ], 0)

        np.subtract(self._ring_buff[:last_row], means, self._std_buff[:last_row])
        np.abs(self._std_buff[:last_row], self._std_buff[:last_row])

        stds = np.mean(self._std_buff[:last_row], 0)
        if (stds == 0).any():
            return 0

        a = 1 / (stds* self._sqrt_2_pi)

        b = np.exp(- (features - means) ** 2  / (2 * stds ** 2))

        likelihoods =  a * b

        if np.any(likelihoods==0):
            return 0
        #print features, means
        logls = np.sum(np.log10(likelihoods)) / len(likelihoods)
        return -1.0 * logls 

def _pre_process_input_minimal(self, img, mask, t, darker_fg=True):
        blur_rad = int(self._object_expected_size * np.max(img.shape) / 2.0)

        if blur_rad % 2 == 0:
            blur_rad += 1

        if self._buff_grey is None:
            self._buff_grey = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
            if mask is None:
                mask = np.ones_like(self._buff_grey) * 255

        cv2.cvtColor(img,cv2.COLOR_BGR2GRAY, self._buff_grey)
        # cv2.imshow("dbg",self._buff_grey)
        cv2.GaussianBlur(self._buff_grey,(blur_rad,blur_rad),1.2, self._buff_grey)
        if darker_fg:
            cv2.subtract(255, self._buff_grey, self._buff_grey)

        #
        mean = cv2.mean(self._buff_grey, mask)

        scale = 128. / mean[0]

        cv2.multiply(self._buff_grey, scale, dst = self._buff_grey)


        if mask is not None:
            cv2.bitwise_and(self._buff_grey, mask, self._buff_grey)
            return self._buff_grey 

def postprocess(score_map, geo_map, score_map_thresh=0.01, box_thresh=0.01, nms_thres=0.2):
    """     Removal of boxes according to various thresholds
    Args:
            *   score_map: confidence score of boxes
            *   geo_map: coordinates of boxes
            *   score_map_thresh: Threshold to filter according to score map
            *   box_thresh: Threshold to filter resultant boxes
            *   nms_thres: Threshold to filter according to Non-Maximum Suppression
    Workflow:
            *   sort the text boxes in accordance to the y axis
            *   restore_rectangle is called to handle tilted boxes
            *   Locality-Aware Non-Maximum Suppression (LANMS) is used to filter
                out overlapping boxes
            *   Average Score Map is used to filter out more boxes
            *   box_thresh is finally used to further filter out boxes
    Returns:
            *   Rezised Image along  with resizing factor
    """

    logger.info(msg="postprocess called")
    if len(score_map.shape) == 4:
        score_map = score_map[0, :, :, 0]
        geo_map = geo_map[0, :, :, ]
    xy_text = np.argwhere(score_map > score_map_thresh)
    xy_text = xy_text[np.argsort(xy_text[:, 0])]
    text_box_restored = restore_rectangle(xy_text[:, ::-1] * 4, geo_map[xy_text[:, 0], xy_text[:, 1], :])
    boxes = np.zeros((text_box_restored.shape[0], 9), dtype=np.float32)
    boxes[:, :8] = text_box_restored.reshape((-1, 8))
    boxes[:, 8] = score_map[xy_text[:, 0], xy_text[:, 1]]

    boxes = lanms.merge_quadrangle_n9(boxes.astype('float32'), nms_thres)

    if boxes.shape[0] == 0:
        return None

    for i, box in enumerate(boxes):
        mask = np.zeros_like(score_map, dtype=np.uint8)
        cv2.fillPoly(mask, box[:8].reshape((-1, 4, 2)).astype(np.int32) // 4, 1)
        boxes[i, 8] = cv2.mean(score_map, mask)[0]
    boxes = boxes[boxes[:, 8] > box_thresh]

    return boxes 

def detect_single_scale(score_map, geo_map, score_map_thresh, nms_thres, box_thresh, timer):
    if len(score_map.shape) == 4:
        score_map = score_map[0, :, :, 0]
        geo_map = geo_map[0, :, :, ]
    # filter the score map
    xy_text = np.argwhere(score_map > score_map_thresh)
    # sort the text boxes via the y axis
    xy_text = xy_text[np.argsort(xy_text[:, 0])]
    # restore
    start = time.time()
    # xy_text[:, ::-1]*4 满足条件的pixel的坐标
    # geo_map[xy_text[:, 0], xy_text[:, 1], :] 得到对应点到ｂｏｕｎｄｉｎｇ　ｂｏｘ　的距离
    text_box_restored = restore_rectangle(xy_text[:, ::-1], geo_map[xy_text[:, 0], xy_text[:, 1], :])  # N*4*2
    print('{} text boxes before nms'.format(text_box_restored.shape[0]))
    boxes = np.zeros((text_box_restored.shape[0], 9), dtype=np.float32)
    boxes[:, :8] = text_box_restored.reshape((-1, 8))
    boxes[:, 8] = score_map[xy_text[:, 0], xy_text[:, 1]]
    timer['restore'] = time.time() - start

    # Modify Start
    # 我们以bounding box内的平均值作为nms的标准而不是一个点的值
    # new_boxes = np.copy(boxes)
    # for i, box in enumerate(new_boxes):
    #     mask = np.zeros_like(score_map, dtype=np.uint8)
    #     cv2.fillPoly(mask, box[:8].reshape((-1, 4, 2)).astype(np.int32), 1)
    #     new_boxes[i, 8] = cv2.mean(score_map, mask)[0]
    # end


    # nms part
    start = time.time()
    # boxes = nms_locality.nms_locality(boxes.astype(np.float64), nms_thres)
    boxes = lanms.merge_quadrangle_n9(boxes.astype('float32'), nms_thres)
    # boxes = lanms.merge_quadrangle_n9(new_boxes.astype('float32'), nms_thres)

    timer['nms'] = time.time() - start

    if boxes.shape[0] == 0:
        return None, timer

    # here we filter some low score boxes by the average score map, this is different from the orginal paper
    for i, box in enumerate(boxes):
        mask = np.zeros_like(score_map, dtype=np.uint8)
        cv2.fillPoly(mask, box[:8].reshape((-1, 4, 2)).astype(np.int32), 1)
        boxes[i, 8] = cv2.mean(score_map, mask)[0]
    boxes = boxes[boxes[:, 8] > box_thresh]
    return boxes 

def detect(score_map, geo_map, timer, score_map_thresh=0.8, box_thresh=0.1, nms_thres=0.2):
    '''
    restore text boxes from score map and geo map
    :param score_map:
    :param geo_map:[W,H,5]
    :param timer:
    :param score_map_thresh: threshhold for score map
    :param box_thresh: threshhold for boxes
    :param nms_thres: threshold for nms
    :return:
    '''
    if len(score_map.shape) == 4:
        score_map = score_map[0, :, :, 0]
        geo_map = geo_map[0, :, :, ]
    # filter the score map
    xy_text = np.argwhere(score_map > score_map_thresh)
    # sort the text boxes via the y axis
    xy_text = xy_text[np.argsort(xy_text[:, 0])]
    # restore
    start = time.time()
    # xy_text[:, ::-1]*4 满足条件的pixel的坐标
    # geo_map[xy_text[:, 0], xy_text[:, 1], :] 得到对应点到ｂｏｕｎｄｉｎｇ　ｂｏｘ　的距离
    text_box_restored = restore_rectangle(xy_text[:, ::-1], geo_map[xy_text[:, 0], xy_text[:, 1], :]) # N*4*2
    print('{} text boxes before nms'.format(text_box_restored.shape[0]))
    boxes = np.zeros((text_box_restored.shape[0], 9), dtype=np.float32)
    boxes[:, :8] = text_box_restored.reshape((-1, 8))
    boxes[:, 8] = score_map[xy_text[:, 0], xy_text[:, 1]]
    timer['restore'] = time.time() - start

    # Modify Start
    # 我们以bounding box内的平均值作为nms的标准而不是一个点的值
    new_boxes = np.copy(boxes)
    for i, box in enumerate(new_boxes):
        mask = np.zeros_like(score_map, dtype=np.uint8)
        cv2.fillPoly(mask, box[:8].reshape((-1, 4, 2)).astype(np.int32), 1)
        new_boxes[i, 8] = cv2.mean(score_map, mask)[0]
    # end

    # nms part
    start = time.time()
    # boxes = nms_locality.nms_locality(boxes.astype(np.float64), nms_thres)
    # boxes = lanms.merge_quadrangle_n9(boxes.astype('float32'), nms_thres)
    boxes = lanms.merge_quadrangle_n9(new_boxes.astype('float32'), nms_thres)
    timer['nms'] = time.time() - start

    if boxes.shape[0] == 0:
        return None, timer

    # here we filter some low score boxes by the average score map, this is different from the orginal paper
    for i, box in enumerate(boxes):
        mask = np.zeros_like(score_map, dtype=np.uint8)
        cv2.fillPoly(mask, box[:8].reshape((-1, 4, 2)).astype(np.int32), 1)
        boxes[i, 8] = cv2.mean(score_map, mask)[0]
    boxes = boxes[boxes[:, 8] > box_thresh]

    return boxes, timer 

def detect(score_map, geo_map, timer, score_map_thresh=0.8, box_thresh=0.1, nms_thres=0.2):
    '''
    restore text boxes from score map and geo map
    :param score_map:
    :param geo_map:
    :param timer:
    :param score_map_thresh: threshhold for score map
    :param box_thresh: threshhold for boxes
    :param nms_thres: threshold for nms
    :return:
    '''
    if len(score_map.shape) == 4:
        score_map = score_map[0, :, :, 0]
        geo_map = geo_map[0, :, :, ]
    # filter the score map
    xy_text = np.argwhere(score_map > score_map_thresh)
    # sort the text boxes via the y axis
    xy_text = xy_text[np.argsort(xy_text[:, 0])]
    # restore
    start = time.time()
    text_box_restored = restore_rectangle(xy_text[:, ::-1]*4, geo_map[xy_text[:, 0], xy_text[:, 1], :]) # N*4*2
    print('{} text boxes before nms'.format(text_box_restored.shape[0]))
    boxes = np.zeros((text_box_restored.shape[0], 9), dtype=np.float32)
    boxes[:, :8] = text_box_restored.reshape((-1, 8))
    boxes[:, 8] = score_map[xy_text[:, 0], xy_text[:, 1]]
    timer['restore'] = time.time() - start
    # nms part
    start = time.time()
    # boxes = nms_locality.nms_locality(boxes.astype(np.float64), nms_thres)
    boxes = lanms.merge_quadrangle_n9(boxes.astype('float32'), nms_thres)
    timer['nms'] = time.time() - start

    if boxes.shape[0] == 0:
        return None, timer

    # here we filter some low score boxes by the average score map, this is different from the orginal paper
    for i, box in enumerate(boxes):
        mask = np.zeros_like(score_map, dtype=np.uint8)
        cv2.fillPoly(mask, box[:8].reshape((-1, 4, 2)).astype(np.int32) // 4, 1)
        boxes[i, 8] = cv2.mean(score_map, mask)[0]
    boxes = boxes[boxes[:, 8] > box_thresh]

    return boxes, timer 

def detect(score_map, geo_map, timer, score_map_thresh=0.8, box_thresh=0.1, nms_thres=0.2):
    '''
    restore text boxes from score map and geo map
    :param score_map:
    :param geo_map:[W,H,5]
    :param timer:
    :param score_map_thresh: threshhold for score map
    :param box_thresh: threshhold for boxes
    :param nms_thres: threshold for nms
    :return:
    '''
    if len(score_map.shape) == 4:
        score_map = score_map[0, :, :, 0]
        geo_map = geo_map[0, :, :, ]
    # filter the score map
    xy_text = np.argwhere(score_map > score_map_thresh)
    # sort the text boxes via the y axis
    xy_text = xy_text[np.argsort(xy_text[:, 0])]
    # restore
    start = time.time()
    # xy_text[:, ::-1]*4 满足条件的pixel的坐标
    # geo_map[xy_text[:, 0], xy_text[:, 1], :] 得到对应点到ｂｏｕｎｄｉｎｇ　ｂｏｘ　的距离
    text_box_restored = restore_rectangle(xy_text[:, ::-1], geo_map[xy_text[:, 0], xy_text[:, 1], :]) # N*4*2
    print('{} text boxes before nms'.format(text_box_restored.shape[0]))
    boxes = np.zeros((text_box_restored.shape[0], 9), dtype=np.float32)
    boxes[:, :8] = text_box_restored.reshape((-1, 8))
    boxes[:, 8] = score_map[xy_text[:, 0], xy_text[:, 1]]
    timer['restore'] = time.time() - start
    # nms part
    start = time.time()
    # boxes = nms_locality.nms_locality(boxes.astype(np.float64), nms_thres)
    boxes = lanms.merge_quadrangle_n9(boxes.astype('float32'), nms_thres)
    timer['nms'] = time.time() - start

    if boxes.shape[0] == 0:
        return None, timer

    # here we filter some low score boxes by the average score map, this is different from the orginal paper
    for i, box in enumerate(boxes):
        mask = np.zeros_like(score_map, dtype=np.uint8)
        cv2.fillPoly(mask, box[:8].reshape((-1, 4, 2)).astype(np.int32), 1)
        boxes[i, 8] = cv2.mean(score_map, mask)[0]
    boxes = boxes[boxes[:, 8] > box_thresh]

    return boxes, timer 

def align_face(input, preds, canonical_vertices, target_size=(318,361)):

    #get (x,y) coordinates
    canonical_vertices = canonical_vertices[:,:2]
    preds = preds[:,:2]
    front_vertices = canonical_vertices #front_vertices[:,:2]

    #get (x,y,1) coordinates
    pts0, pts1, pts2 = canonical_vertices, preds, front_vertices
    pts0_homo = np.hstack((pts0, np.ones([pts0.shape[0],1]))) #n x 4
    pts1_homo = np.hstack((pts1, np.ones([pts1.shape[0],1]))) #n x 4
    pts2_homo = np.hstack((pts2, np.ones([pts2.shape[0],1]))) #n x 4

    #get 3D transform parameters
    AM_68 = np.linalg.lstsq(pts1_homo, pts2_homo)[0].T # Affine matrix. 3 x 3
    pts1_homo_5 = np.float32([np.mean(pts1_homo[[17,36]], axis=0), np.mean(pts1_homo[[26,45]], axis=0), np.mean(pts1_homo[[30]], axis=0), np.mean(pts1_homo[[48,60]], axis=0), np.mean(pts1_homo[[54,64]], axis=0)])
    pts2_homo_5 = np.float32([np.mean(pts2_homo[[17,36]], axis=0), np.mean(pts2_homo[[26,45]], axis=0), np.mean(pts2_homo[[30]], axis=0), np.mean(pts2_homo[[48,60]], axis=0), np.mean(pts2_homo[[54,64]], axis=0)])
    AM_5 = np.linalg.lstsq(pts1_homo_5, pts2_homo_5)[0].T # Affine matrix. 3 x 3
    pts1_homo_eye = np.float32([np.mean(pts1_homo[[36]], axis=0), np.mean(pts1_homo[[39]], axis=0), np.mean(pts1_homo[[8]], axis=0), np.mean(pts1_homo[[42]], axis=0), np.mean(pts1_homo[[45]], axis=0)])
    pts2_homo_eye = np.float32([np.mean(pts2_homo[[36]], axis=0), np.mean(pts2_homo[[39]], axis=0), np.mean(pts2_homo[[8]], axis=0), np.mean(pts2_homo[[42]], axis=0), np.mean(pts2_homo[[45]], axis=0)])
    AM_eye = np.linalg.lstsq(pts1_homo_eye, pts2_homo_eye)[0].T # Affine matrix. 3 x 3
    #AM = np.median([AM_68,AM_5,AM_eye],axis=0)
    AM = (1.0*AM_68 + 2.0*AM_5 + 3.0*AM_eye) / (1.0+2.0+3.0)

    #border landmark indices
    x0_index_src = np.where(pts0[:,0]==np.amin(pts0[:,0]))[0][0]
    x1_index_src = np.where(pts0[:,0]==np.amax(pts0[:,0]))[0][0]
    y0_index_src = np.where(pts0[:,1]==np.amin(pts0[:,1]))[0][0]
    y1_index_src = np.where(pts0[:,1]==np.amax(pts0[:,1]))[0][0]

    # (x,y) limits
    x0 = pts0_homo[x0_index_src][0]
    x1 = pts0_homo[x1_index_src][0]
    y0 = pts0_homo[y0_index_src][1]
    y1 = pts0_homo[y1_index_src][1]

    # get affine transformed image
    input_crop = input #[y0:y1, x0:x1, :]
    dst = cv2.warpPerspective(input_crop, AM, target_size, flags=cv2.INTER_LINEAR, borderMode=cv2.BORDER_REFLECT)

    #crop transformed image
    #x0 = max(0, int(x0))
    #x1 = min(dst.shape[1],int(x1))+1
    #y0 = max(0, int(y0))
    #y1 = min(dst.shape[0],int(y1))+1
    #dst_crop = dst[y0:y1, x0:x1,:]
    return dst 

def compute_features(self, img, contour):
        x,y,w,h = cv2.boundingRect(contour)

        if self._roi_img_buff is None or np.any(self._roi_img_buff.shape < img.shape[0:2]) :
            # dynamically reallocate buffer if needed
            self._img_buff_shape[1] =  max(self._img_buff_shape[1],w)
            self._img_buff_shape[0] =  max(self._img_buff_shape[0], h)

            self._roi_img_buff = np.zeros(self._img_buff_shape, np.uint8)
            self._mask_img_buff = np.zeros_like(self._roi_img_buff)

        sub_mask = self._mask_img_buff[0 : h, 0 : w]

        sub_grey = self._roi_img_buff[ 0 : h, 0: w]

        cv2.cvtColor(img[y : y + h, x : x + w, :],cv2.COLOR_BGR2GRAY,sub_grey)
        sub_mask.fill(0)

        cv2.drawContours(sub_mask,[contour],-1, 255,-1,offset=(-x,-y))
        mean_col = cv2.mean(sub_grey, sub_mask)[0]


        (_,_) ,(width,height), angle  = cv2.minAreaRect(contour)
        width, height= max(width,height), min(width,height)
        ar = ((height+1) / (width+1))
        #todo speed should use time
        #
        # if len(self.positions) > 2:
        #
        #     pm, pmm = self._positions[-1],self._positions[-2]
        #     xm, xmm = pm["x"], pmm["x"]
        #     ym, ymm = pm["y"], pmm["y"]
        #
        #     instantaneous_speed = abs(xm + 1j*ym - xmm + 1j*ymm)
        # else:
        #     instantaneous_speed = 0
        # if np.isnan(instantaneous_speed):
        #     instantaneous_speed = 0

        features = np.array([log10(cv2.contourArea(contour) + 1.0),
                            height + 1,
                            #sqrt(ar),
                            #instantaneous_speed +1.0,
                            mean_col +1
                            # 1.0
                             ])

        return features 

def _pre_process_input(self, img, mask, t):

        blur_rad = int(self._object_expected_size * np.max(img.shape) * 2.0)
        if blur_rad % 2 == 0:
            blur_rad += 1


        if self._buff_grey is None:
            self._buff_grey = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
            self._buff_grey_blurred = np.empty_like(self._buff_grey)
            # self._buff_grey_blurred = np.empty_like(self._buff_grey)
            if mask is None:
                mask = np.ones_like(self._buff_grey) * 255

            mask_conv = cv2.blur(mask,(blur_rad, blur_rad))

            self._buff_convolved_mask  = (1/255.0 *  mask_conv.astype(np.float32))


        cv2.cvtColor(img,cv2.COLOR_BGR2GRAY, self._buff_grey)

        hist = cv2.calcHist([self._buff_grey], [0], None, [256], [0,255]).ravel()
        hist = np.convolve(hist, [1] * 3)
        mode =  np.argmax(hist)

        self._smooth_mode.append(mode)
        self._smooth_mode_tstamp.append(t)

        if len(self._smooth_mode_tstamp) >2 and self._smooth_mode_tstamp[-1] - self._smooth_mode_tstamp[0] > self._smooth_mode_window_dt:
            self._smooth_mode.popleft()
            self._smooth_mode_tstamp.popleft()


        mode = np.mean(list(self._smooth_mode))
        scale = 128. / mode

        # cv2.GaussianBlur(self._buff_grey,(5,5), 1.5,self._buff_grey)

        cv2.multiply(self._buff_grey, scale, dst = self._buff_grey)


        cv2.bitwise_and(self._buff_grey, mask, self._buff_grey)

        cv2.blur(self._buff_grey,(blur_rad, blur_rad), self._buff_grey_blurred)
        #fixme could be optimised
        self._buff_grey_blurred = (self._buff_grey_blurred / self._buff_convolved_mask).astype(np.uint8)


        cv2.absdiff(self._buff_grey, self._buff_grey_blurred, self._buff_grey)

        if mask is not None:
            cv2.bitwise_and(self._buff_grey, mask, self._buff_grey)
            return self._buff_grey