(self, inputs, inference=False)
fprop function that does no proposal filtering
fprop function that does no proposal filtering
fprop function that does no proposal filtering
def fprop(self, inputs, inference=False):
        """
        fprop function that does no proposal filtering
        """
        assert self.inference == inference, \
            "Model was configured for inference={}".format(self.inference)

        # get needed metadata from the dataloader
        (self.im_shape, self.im_scale, self.gt_boxes,
            self.gt_classes, self.num_gt_boxes, _) = self.dataloader.get_metadata_buffers()

        # real H and W need to get in fprop, as every image is different
        real_H = int(np.round(self.im_shape.get()[1] * self._scale))
        real_W = int(np.round(self.im_shape.get()[0] * self._scale))

        if self.rpn_scores_v is None:
            # get output from the RPN network
            # transform the scores and slice the score for the label=1 class
            # shape: (KWH, 1)
            self.rpn_scores_v = self.rpn_obj[0].outputs.reshape((2, -1))[1].T

        if self.bbox_deltas_v is None:
            # transform the bbox deltas, reshape to (4, KHW) then transpose to
            # (KHW, 4) to match the shape of anchors.
            self.bbox_deltas_v = self.rpn_bbox[0].outputs.reshape((4, -1)).T

        # 1. Convert anchors into proposals via bbox transformations
        # store output in proposals buffer
        self._bbox_transform_inv(self._dev_anchors, self.bbox_deltas_v, output=self._proposals)

        # 2. clip predicted boxes to image
        self._clip_boxes(self._proposals, self.im_shape)

        # 3. remove predicted boxes with either height or width < threshold
        # (NOTE: convert min_size to input image scale stored in im_scale)
        keep = self._filter_boxes(self._proposals,
                                  self.min_bbox_size * float(self.im_scale.get()))

        # we set the score of those we discard to -1
        self._scores[:] = (self.rpn_scores_v * keep) - (1 - keep)

        # 4. set the scores to be -1 for the padded area
        # set the scores of all the proposals from the padded area to be -1
        # in order to ignore them after sorting
        scores_np = self._scores.get()
        scores_np.reshape(-1, self._conv_height, self._conv_width)[:, real_H:] = -1
        scores_np.reshape(-1, self._conv_height, self._conv_width)[:, :, real_W:] = -1
        self._scores[:] = self.be.array(scores_np)

        # 5. sort the scores from highest to lowest and take top pre_nms_topN
        top_N_ind = self.get_top_N_index(self._scores, self.pre_nms_N)

        # take top pre_nms_topN (e.g. 12000)
        # (make scores & proposals attributes of layer for unit testing)
        self.dets.fill(0)
        self.dets[:len(top_N_ind), :4] = self._proposals[top_N_ind]
        self.dets[:len(top_N_ind), 4] = self._scores[top_N_ind]

        # 6. apply nms (e.g. threshold = 0.7)
        keep = self.be.nms(self.dets, self.nms_thresh)

        # 7. take post_nms_N (e.g. 2000)
        keep = keep[:self.post_nms_N]
        self.num_proposals = len(keep)

        # for training or debugging, we need to copy these detections to host.
        if self.debug or not inference:
            # make scores & proposals attributes of layer for unit testing
            self.proposals = self.dets[keep, :4].get()
            self.scores = self.dets[keep, -1].get()

        # 8. provide ROIs in the format of [0, x1, y1, x2, y2]
        self.dev_proposals.fill(0)
        self.dev_proposals[:self.num_proposals, 1:] = self.dets[keep, :4]

        # If training, sample the proposals and only propagate those forward
        if not inference:
            # Next, we need to set the target buffers with the class labels,
            # and bbox targets for each roi.
            ((frcn_labels, frcn_labels_mask),
             (frcn_bbtargets, frcn_bbmask)) = self.dataloader.get_target_buffers()

            non_zero_gt_boxes = self.gt_boxes.get()
            num_gt_boxes = self.num_gt_boxes.get()[0][0]
            non_zero_gt_boxes = non_zero_gt_boxes[:num_gt_boxes]

            # Include ground-truth boxes in the set of candidate rois
            all_rois = np.vstack((self.proposals, non_zero_gt_boxes))

            # 1. Compute the overlap of each proposal roi with each ground truth roi
            overlaps = calculate_bb_overlap(all_rois, non_zero_gt_boxes)

            # 2. Use overlaps to compute the gt box each proposal is 'closest' to
            gt_assignment = overlaps.argmax(axis=1)
            max_overlaps = overlaps.max(axis=1)
            labels = self.gt_classes.get()[:num_gt_boxes]
            labels = labels[gt_assignment]

            # Sample num_rois fg and bg indicies based on overlaps with gt bocxes
            keep_inds, fg_rois_this_img = self._sample_fg_bg(max_overlaps)
            # Select sampled values from various arrays:
            labels = labels[keep_inds]
            # Clamp labels for the background RoIs to 0
            labels[fg_rois_this_img:] = 0

            rois = all_rois[keep_inds]
            targets = compute_targets(non_zero_gt_boxes[gt_assignment[keep_inds]], rois)

            targets = (targets - np.array(BBOX_NORMALIZE_MEANS)) / np.array(BBOX_NORMALIZE_STDS)

            num_proposals = rois.shape[0]
            bbox_targets, bbox_inside_weights = \
                self._get_bbox_regression_labels(targets, labels)

            # Load fast-rcnn training labels and bbox targets back to global buffers
            labels_full = self._onehot_labels(labels.ravel())
            frcn_labels[:] = labels_full
            labels_mask = np.zeros((self.num_rois, self.num_classes))
            labels_mask[:num_proposals, :] = 1.0
            frcn_labels_mask[:] = np.ascontiguousarray(labels_mask.T)

            # fcrn_*.shape = (num_classes*4 , 256), so transpose first
            frcn_bbtargets[:] = np.ascontiguousarray(bbox_targets.T)
            frcn_bbmask[:] = np.ascontiguousarray(bbox_inside_weights.T)

            # load the sampled proposals back to device
            rois = np.hstack([np.zeros((num_proposals, 1)), rois])
            rois = np.ascontiguousarray(rois, dtype=np.float32)

            self.dev_proposals_filtered.fill(0)
            self.dev_proposals_filtered[:num_proposals, :] = rois
            self.num_proposals = num_proposals

            # During training, only propagate sampled proposals
            return (inputs, self.dev_proposals_filtered.T)

        # If inference=True, we're testing, so propagate all proposals
        else:
            return (inputs, self.dev_proposals.T)
def fprop ( self , inputs , inference = False ) : assert self . inference == inference , "Model was configured for inference={}" . format ( self . inference ) # get needed metadata from the dataloader ( self . im_shape , self . im_scale , self . gt_boxes , self . gt_classes , self . num_gt_boxes , _ ) = self . dataloader . get_metadata_buffers ( ) # real H and W need to get in fprop, as every image is different real_H = int ( np . round ( self . im_shape . get ( ) [ 1 ] * self . _scale ) ) real_W = int ( np . round ( self . im_shape . get ( ) [ 0 ] * self . _scale ) ) if self . rpn_scores_v is None : # get output from the RPN network # transform the scores and slice the score for the label=1 class # shape: (KWH, 1) self . rpn_scores_v = self . rpn_obj [ 0 ] . outputs . reshape ( ( 2 , - 1 ) ) [ 1 ] . T if self . bbox_deltas_v is None : # transform the bbox deltas, reshape to (4, KHW) then transpose to # (KHW, 4) to match the shape of anchors. self . bbox_deltas_v = self . rpn_bbox [ 0 ] . outputs . reshape ( ( 4 , - 1 ) ) . T # 1. Convert anchors into proposals via bbox transformations # store output in proposals buffer self . _bbox_transform_inv ( self . _dev_anchors , self . bbox_deltas_v , output = self . _proposals ) # 2. clip predicted boxes to image self . _clip_boxes ( self . _proposals , self . im_shape ) # 3. remove predicted boxes with either height or width < threshold # (NOTE: convert min_size to input image scale stored in im_scale) keep = self . _filter_boxes ( self . _proposals , self . min_bbox_size * float ( self . im_scale . get ( ) ) ) # we set the score of those we discard to -1 self . _scores [ : ] = ( self . rpn_scores_v * keep ) - ( 1 - keep ) # 4. set the scores to be -1 for the padded area # set the scores of all the proposals from the padded area to be -1 # in order to ignore them after sorting scores_np = self . _scores . get ( ) scores_np . reshape ( - 1 , self . _conv_height , self . _conv_width ) [ : , real_H : ] = - 1 scores_np . reshape ( - 1 , self . _conv_height , self . _conv_width ) [ : , : , real_W : ] = - 1 self . _scores [ : ] = self . be . array ( scores_np ) # 5. sort the scores from highest to lowest and take top pre_nms_topN top_N_ind = self . get_top_N_index ( self . _scores , self . pre_nms_N ) # take top pre_nms_topN (e.g. 12000) # (make scores & proposals attributes of layer for unit testing) self . dets . fill ( 0 ) self . dets [ : len ( top_N_ind ) , : 4 ] = self . _proposals [ top_N_ind ] self . dets [ : len ( top_N_ind ) , 4 ] = self . _scores [ top_N_ind ] # 6. apply nms (e.g. threshold = 0.7) keep = self . be . nms ( self . dets , self . nms_thresh ) # 7. take post_nms_N (e.g. 2000) keep = keep [ : self . post_nms_N ] self . num_proposals = len ( keep ) # for training or debugging, we need to copy these detections to host. if self . debug or not inference : # make scores & proposals attributes of layer for unit testing self . proposals = self . dets [ keep , : 4 ] . get ( ) self . scores = self . dets [ keep , - 1 ] . get ( ) # 8. provide ROIs in the format of [0, x1, y1, x2, y2] self . dev_proposals . fill ( 0 ) self . dev_proposals [ : self . num_proposals , 1 : ] = self . dets [ keep , : 4 ] # If training, sample the proposals and only propagate those forward if not inference : # Next, we need to set the target buffers with the class labels, # and bbox targets for each roi. ( ( frcn_labels , frcn_labels_mask ) , ( frcn_bbtargets , frcn_bbmask ) ) = self . dataloader . get_target_buffers ( ) non_zero_gt_boxes = self . gt_boxes . get ( ) num_gt_boxes = self . num_gt_boxes . get ( ) [ 0 ] [ 0 ] non_zero_gt_boxes = non_zero_gt_boxes [ : num_gt_boxes ] # Include ground-truth boxes in the set of candidate rois all_rois = np . vstack ( ( self . proposals , non_zero_gt_boxes ) ) # 1. Compute the overlap of each proposal roi with each ground truth roi overlaps = calculate_bb_overlap ( all_rois , non_zero_gt_boxes ) # 2. Use overlaps to compute the gt box each proposal is 'closest' to gt_assignment = overlaps . argmax ( axis = 1 ) max_overlaps = overlaps . max ( axis = 1 ) labels = self . gt_classes . get ( ) [ : num_gt_boxes ] labels = labels [ gt_assignment ] # Sample num_rois fg and bg indicies based on overlaps with gt bocxes keep_inds , fg_rois_this_img = self . _sample_fg_bg ( max_overlaps ) # Select sampled values from various arrays: labels = labels [ keep_inds ] # Clamp labels for the background RoIs to 0 labels [ fg_rois_this_img : ] = 0 rois = all_rois [ keep_inds ] targets = compute_targets ( non_zero_gt_boxes [ gt_assignment [ keep_inds ] ] , rois ) targets = ( targets - np . array ( BBOX_NORMALIZE_MEANS ) ) / np . array ( BBOX_NORMALIZE_STDS ) num_proposals = rois . shape [ 0 ] bbox_targets , bbox_inside_weights = self . _get_bbox_regression_labels ( targets , labels ) # Load fast-rcnn training labels and bbox targets back to global buffers labels_full = self . _onehot_labels ( labels . ravel ( ) ) frcn_labels [ : ] = labels_full labels_mask = np . zeros ( ( self . num_rois , self . num_classes ) ) labels_mask [ : num_proposals , : ] = 1.0 frcn_labels_mask [ : ] = np . ascontiguousarray ( labels_mask . T ) # fcrn_*.shape = (num_classes*4 , 256), so transpose first frcn_bbtargets [ : ] = np . ascontiguousarray ( bbox_targets . T ) frcn_bbmask [ : ] = np . ascontiguousarray ( bbox_inside_weights . T ) # load the sampled proposals back to device rois = np . hstack ( [ np . zeros ( ( num_proposals , 1 ) ) , rois ] ) rois = np . ascontiguousarray ( rois , dtype = np . float32 ) self . dev_proposals_filtered . fill ( 0 ) self . dev_proposals_filtered [ : num_proposals , : ] = rois self . num_proposals = num_proposals # During training, only propagate sampled proposals return ( inputs , self . dev_proposals_filtered . T ) # If inference=True, we're testing, so propagate all proposals else : return ( inputs , self . dev_proposals . T )