(gtFilePath, submFilePath, evaluationParams)
return resDict
Method evaluate_method: evaluate method and returns the results
        Results. Dictionary with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
Method evaluate_method: evaluate method and returns the results
        Results. Dictionary with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
Method evaluate_method : evaluate method and returns the results Results . Dictionary with the following values : - method ( required ) Global method metrics . Ex : { Precision : 0 . 8 Recall : 0 . 9 } - samples ( optional ) Per sample metrics . Ex : { sample1 : { Precision : 0 . 8 Recall : 0 . 9 } sample2 : { Precision : 0 . 8 Recall : 0 . 9 }
def evaluate_method(gtFilePath, submFilePath, evaluationParams):
    """
    Method evaluate_method: evaluate method and returns the results
        Results. Dictionary with the following values:
        - method (required)  Global method metrics. Ex: { 'Precision':0.8,'Recall':0.9 }
        - samples (optional) Per sample metrics. Ex: {'sample1' : { 'Precision':0.8,'Recall':0.9 } , 'sample2' : { 'Precision':0.8,'Recall':0.9 }
    """

    def polygon_from_points(points):
        """
        Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4
        """
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(points[0])
        resBoxes[0, 4] = int(points[1])
        resBoxes[0, 1] = int(points[2])
        resBoxes[0, 5] = int(points[3])
        resBoxes[0, 2] = int(points[4])
        resBoxes[0, 6] = int(points[5])
        resBoxes[0, 3] = int(points[6])
        resBoxes[0, 7] = int(points[7])
        pointMat = resBoxes[0].reshape([2, 4]).T
        return plg.Polygon(pointMat)

    def rectangle_to_polygon(rect):
        resBoxes = np.empty([1, 8], dtype='int32')
        resBoxes[0, 0] = int(rect.xmin)
        resBoxes[0, 4] = int(rect.ymax)
        resBoxes[0, 1] = int(rect.xmin)
        resBoxes[0, 5] = int(rect.ymin)
        resBoxes[0, 2] = int(rect.xmax)
        resBoxes[0, 6] = int(rect.ymin)
        resBoxes[0, 3] = int(rect.xmax)
        resBoxes[0, 7] = int(rect.ymax)

        pointMat = resBoxes[0].reshape([2, 4]).T

        return plg.Polygon(pointMat)

    def rectangle_to_points(rect):
        points = [int(rect.xmin), int(rect.ymax), int(rect.xmax), int(rect.ymax), int(rect.xmax), int(rect.ymin),
                  int(rect.xmin), int(rect.ymin)]
        return points

    def get_union(pD, pG):
        areaA = pD.area();
        areaB = pG.area();
        return areaA + areaB - get_intersection(pD, pG);

    def get_intersection_over_union(pD, pG):
        try:
            return get_intersection(pD, pG) / get_union(pD, pG);
        except:
            return 0

    def get_intersection(pD, pG):
        pInt = pD & pG
        if len(pInt) == 0:
            return 0
        return pInt.area()

    def compute_ap(confList, matchList, numGtCare):
        correct = 0
        AP = 0
        if len(confList) > 0:
            confList = np.array(confList)
            matchList = np.array(matchList)
            sorted_ind = np.argsort(-confList)
            confList = confList[sorted_ind]
            matchList = matchList[sorted_ind]
            for n in range(len(confList)):
                match = matchList[n]
                if match:
                    correct += 1
                    AP += float(correct) / (n + 1)

            if numGtCare > 0:
                AP /= numGtCare

        return AP

    perSampleMetrics = {}

    matchedSum = 0

    Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')

    gt = rrc_evaluation_funcs.load_folder_file(gtFilePath, evaluationParams['GT_SAMPLE_NAME_2_ID'])
    subm = rrc_evaluation_funcs.load_folder_file(submFilePath, evaluationParams['DET_SAMPLE_NAME_2_ID'], True)

    numGlobalCareGt = 0;
    numGlobalCareDet = 0;

    arrGlobalConfidences = [];
    arrGlobalMatches = [];

    for resFile in gt:

        gtFile = gt[resFile]  # rrc_evaluation_funcs.decode_utf8(gt[resFile])
        recall = 0
        precision = 0
        hmean = 0

        detMatched = 0

        iouMat = np.empty([1, 1])

        gtPols = []
        detPols = []

        gtPolPoints = []
        detPolPoints = []

        # Array of Ground Truth Polygons' keys marked as don't Care
        gtDontCarePolsNum = []
        # Array of Detected Polygons' matched with a don't Care GT
        detDontCarePolsNum = []

        pairs = []
        detMatchedNums = []

        arrSampleConfidences = [];
        arrSampleMatch = [];
        sampleAP = 0;

        evaluationLog = ""

        pointsList, _, transcriptionsList = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(gtFile,
                                                                                                       evaluationParams[
                                                                                                           'CRLF'],
                                                                                                       evaluationParams[
                                                                                                           'LTRB'],
                                                                                                       True, False)
        for n in range(len(pointsList)):
            points = pointsList[n]
            transcription = transcriptionsList[n]
            dontCare = transcription == "###"
            if evaluationParams['LTRB']:
                gtRect = Rectangle(*points)
                gtPol = rectangle_to_polygon(gtRect)
            else:
                gtPol = polygon_from_points(points)
            gtPols.append(gtPol)
            gtPolPoints.append(points)
            if dontCare:
                gtDontCarePolsNum.append(len(gtPols) - 1)

        evaluationLog += "GT polygons: " + str(len(gtPols)) + (
            " (" + str(len(gtDontCarePolsNum)) + " don't care)\n" if len(gtDontCarePolsNum) > 0 else "\n")

        if resFile in subm:

            detFile = subm[resFile]  # rrc_evaluation_funcs.decode_utf8(subm[resFile])

            pointsList, confidencesList, _ = rrc_evaluation_funcs.get_tl_line_values_from_file_contents(detFile,
                                                                                                        evaluationParams[
                                                                                                            'CRLF'],
                                                                                                        evaluationParams[
                                                                                                            'LTRB'],
                                                                                                        False,
                                                                                                        evaluationParams[
                                                                                                            'CONFIDENCES'])
            for n in range(len(pointsList)):
                points = pointsList[n]

                if evaluationParams['LTRB']:
                    detRect = Rectangle(*points)
                    detPol = rectangle_to_polygon(detRect)
                else:
                    detPol = polygon_from_points(points)
                detPols.append(detPol)
                detPolPoints.append(points)
                if len(gtDontCarePolsNum) > 0:
                    for dontCarePol in gtDontCarePolsNum:
                        dontCarePol = gtPols[dontCarePol]
                        intersected_area = get_intersection(dontCarePol, detPol)
                        pdDimensions = detPol.area()
                        precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions
                        if (precision > evaluationParams['AREA_PRECISION_CONSTRAINT']):
                            detDontCarePolsNum.append(len(detPols) - 1)
                            break

            evaluationLog += "DET polygons: " + str(len(detPols)) + (
                " (" + str(len(detDontCarePolsNum)) + " don't care)\n" if len(detDontCarePolsNum) > 0 else "\n")

            if len(gtPols) > 0 and len(detPols) > 0:
                # Calculate IoU and precision matrixs
                outputShape = [len(gtPols), len(detPols)]
                iouMat = np.empty(outputShape)
                gtRectMat = np.zeros(len(gtPols), np.int8)
                detRectMat = np.zeros(len(detPols), np.int8)
                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        pG = gtPols[gtNum]
                        pD = detPols[detNum]
                        iouMat[gtNum, detNum] = get_intersection_over_union(pD, pG)

                for gtNum in range(len(gtPols)):
                    for detNum in range(len(detPols)):
                        if gtRectMat[gtNum] == 0 and detRectMat[
                            detNum] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum:
                            if iouMat[gtNum, detNum] > evaluationParams['IOU_CONSTRAINT']:
                                gtRectMat[gtNum] = 1
                                detRectMat[detNum] = 1
                                detMatched += 1
                                pairs.append({'gt': gtNum, 'det': detNum})
                                detMatchedNums.append(detNum)
                                evaluationLog += "Match GT #" + str(gtNum) + " with Det #" + str(detNum) + "\n"

            if evaluationParams['CONFIDENCES']:
                for detNum in range(len(detPols)):
                    if detNum not in detDontCarePolsNum:
                        # we exclude the don't care detections
                        match = detNum in detMatchedNums

                        arrSampleConfidences.append(confidencesList[detNum])
                        arrSampleMatch.append(match)

                        arrGlobalConfidences.append(confidencesList[detNum]);
                        arrGlobalMatches.append(match);

        numGtCare = (len(gtPols) - len(gtDontCarePolsNum))
        numDetCare = (len(detPols) - len(detDontCarePolsNum))
        if numGtCare == 0:
            recall = float(1)
            precision = float(0) if numDetCare > 0 else float(1)
            sampleAP = precision
        else:
            recall = float(detMatched) / numGtCare
            precision = 0 if numDetCare == 0 else float(detMatched) / numDetCare
            if evaluationParams['CONFIDENCES'] and evaluationParams['PER_SAMPLE_RESULTS']:
                sampleAP = compute_ap(arrSampleConfidences, arrSampleMatch, numGtCare)

        hmean = 0 if (precision + recall) == 0 else 2.0 * precision * recall / (precision + recall)

        matchedSum += detMatched
        numGlobalCareGt += numGtCare
        numGlobalCareDet += numDetCare

        if evaluationParams['PER_SAMPLE_RESULTS']:
            perSampleMetrics[resFile] = {
                'precision': precision,
                'recall': recall,
                'hmean': hmean,
                'pairs': pairs,
                'AP': sampleAP,
                'iouMat': [] if len(detPols) > 100 else iouMat.tolist(),
                'gtPolPoints': gtPolPoints,
                'detPolPoints': detPolPoints,
                'gtDontCare': gtDontCarePolsNum,
                'detDontCare': detDontCarePolsNum,
                'evaluationParams': evaluationParams,
                'evaluationLog': evaluationLog
            }

    # Compute MAP and MAR
    AP = 0
    if evaluationParams['CONFIDENCES']:
        AP = compute_ap(arrGlobalConfidences, arrGlobalMatches, numGlobalCareGt)

    methodRecall = 0 if numGlobalCareGt == 0 else float(matchedSum) / numGlobalCareGt
    methodPrecision = 0 if numGlobalCareDet == 0 else float(matchedSum) / numGlobalCareDet
    methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / (
            methodRecall + methodPrecision)

    methodMetrics = {'precision': methodPrecision, 'recall': methodRecall, 'hmean': methodHmean, 'AP': AP}

    resDict = {'calculated': True, 'Message': '', 'method': methodMetrics, 'per_sample': perSampleMetrics}

    return resDict;
def evaluate_method ( gtFilePath , submFilePath , evaluationParams ) : def polygon_from_points ( points ) : """
        Returns a Polygon object to use with the Polygon2 class from a list of 8 points: x1,y1,x2,y2,x3,y3,x4,y4
        """ resBoxes = np . empty ( [ 1 , 8 ] , dtype = 'int32' ) resBoxes [ 0 , 0 ] = int ( points [ 0 ] ) resBoxes [ 0 , 4 ] = int ( points [ 1 ] ) resBoxes [ 0 , 1 ] = int ( points [ 2 ] ) resBoxes [ 0 , 5 ] = int ( points [ 3 ] ) resBoxes [ 0 , 2 ] = int ( points [ 4 ] ) resBoxes [ 0 , 6 ] = int ( points [ 5 ] ) resBoxes [ 0 , 3 ] = int ( points [ 6 ] ) resBoxes [ 0 , 7 ] = int ( points [ 7 ] ) pointMat = resBoxes [ 0 ] . reshape ( [ 2 , 4 ] ) . T return plg . Polygon ( pointMat ) def rectangle_to_polygon ( rect ) : resBoxes = np . empty ( [ 1 , 8 ] , dtype = 'int32' ) resBoxes [ 0 , 0 ] = int ( rect . xmin ) resBoxes [ 0 , 4 ] = int ( rect . ymax ) resBoxes [ 0 , 1 ] = int ( rect . xmin ) resBoxes [ 0 , 5 ] = int ( rect . ymin ) resBoxes [ 0 , 2 ] = int ( rect . xmax ) resBoxes [ 0 , 6 ] = int ( rect . ymin ) resBoxes [ 0 , 3 ] = int ( rect . xmax ) resBoxes [ 0 , 7 ] = int ( rect . ymax ) pointMat = resBoxes [ 0 ] . reshape ( [ 2 , 4 ] ) . T return plg . Polygon ( pointMat ) def rectangle_to_points ( rect ) : points = [ int ( rect . xmin ) , int ( rect . ymax ) , int ( rect . xmax ) , int ( rect . ymax ) , int ( rect . xmax ) , int ( rect . ymin ) , int ( rect . xmin ) , int ( rect . ymin ) ] return points def get_union ( pD , pG ) : areaA = pD . area ( ) areaB = pG . area ( ) return areaA + areaB - get_intersection ( pD , pG ) def get_intersection_over_union ( pD , pG ) : try : return get_intersection ( pD , pG ) / get_union ( pD , pG ) except : return 0 def get_intersection ( pD , pG ) : pInt = pD & pG if len ( pInt ) == 0 : return 0 return pInt . area ( ) def compute_ap ( confList , matchList , numGtCare ) : correct = 0 AP = 0 if len ( confList ) > 0 : confList = np . array ( confList ) matchList = np . array ( matchList ) sorted_ind = np . argsort ( - confList ) confList = confList [ sorted_ind ] matchList = matchList [ sorted_ind ] for n in range ( len ( confList ) ) : match = matchList [ n ] if match : correct += 1 AP += float ( correct ) / ( n + 1 ) if numGtCare > 0 : AP /= numGtCare return AP perSampleMetrics = { } matchedSum = 0 Rectangle = namedtuple ( 'Rectangle' , 'xmin ymin xmax ymax' ) gt = rrc_evaluation_funcs . load_folder_file ( gtFilePath , evaluationParams [ 'GT_SAMPLE_NAME_2_ID' ] ) subm = rrc_evaluation_funcs . load_folder_file ( submFilePath , evaluationParams [ 'DET_SAMPLE_NAME_2_ID' ] , True ) numGlobalCareGt = 0 numGlobalCareDet = 0 arrGlobalConfidences = [ ] arrGlobalMatches = [ ] for resFile in gt : gtFile = gt [ resFile ] # rrc_evaluation_funcs.decode_utf8(gt[resFile]) recall = 0 precision = 0 hmean = 0 detMatched = 0 iouMat = np . empty ( [ 1 , 1 ] ) gtPols = [ ] detPols = [ ] gtPolPoints = [ ] detPolPoints = [ ] # Array of Ground Truth Polygons' keys marked as don't Care gtDontCarePolsNum = [ ] # Array of Detected Polygons' matched with a don't Care GT detDontCarePolsNum = [ ] pairs = [ ] detMatchedNums = [ ] arrSampleConfidences = [ ] arrSampleMatch = [ ] sampleAP = 0 evaluationLog = "" pointsList , _ , transcriptionsList = rrc_evaluation_funcs . get_tl_line_values_from_file_contents ( gtFile , evaluationParams [ 'CRLF' ] , evaluationParams [ 'LTRB' ] , True , False ) for n in range ( len ( pointsList ) ) : points = pointsList [ n ] transcription = transcriptionsList [ n ] dontCare = transcription == "###" if evaluationParams [ 'LTRB' ] : gtRect = Rectangle ( * points ) gtPol = rectangle_to_polygon ( gtRect ) else : gtPol = polygon_from_points ( points ) gtPols . append ( gtPol ) gtPolPoints . append ( points ) if dontCare : gtDontCarePolsNum . append ( len ( gtPols ) - 1 ) evaluationLog += "GT polygons: " + str ( len ( gtPols ) ) + ( " (" + str ( len ( gtDontCarePolsNum ) ) + " don't care)\n" if len ( gtDontCarePolsNum ) > 0 else "\n" ) if resFile in subm : detFile = subm [ resFile ] # rrc_evaluation_funcs.decode_utf8(subm[resFile]) pointsList , confidencesList , _ = rrc_evaluation_funcs . get_tl_line_values_from_file_contents ( detFile , evaluationParams [ 'CRLF' ] , evaluationParams [ 'LTRB' ] , False , evaluationParams [ 'CONFIDENCES' ] ) for n in range ( len ( pointsList ) ) : points = pointsList [ n ] if evaluationParams [ 'LTRB' ] : detRect = Rectangle ( * points ) detPol = rectangle_to_polygon ( detRect ) else : detPol = polygon_from_points ( points ) detPols . append ( detPol ) detPolPoints . append ( points ) if len ( gtDontCarePolsNum ) > 0 : for dontCarePol in gtDontCarePolsNum : dontCarePol = gtPols [ dontCarePol ] intersected_area = get_intersection ( dontCarePol , detPol ) pdDimensions = detPol . area ( ) precision = 0 if pdDimensions == 0 else intersected_area / pdDimensions if ( precision > evaluationParams [ 'AREA_PRECISION_CONSTRAINT' ] ) : detDontCarePolsNum . append ( len ( detPols ) - 1 ) break evaluationLog += "DET polygons: " + str ( len ( detPols ) ) + ( " (" + str ( len ( detDontCarePolsNum ) ) + " don't care)\n" if len ( detDontCarePolsNum ) > 0 else "\n" ) if len ( gtPols ) > 0 and len ( detPols ) > 0 : # Calculate IoU and precision matrixs outputShape = [ len ( gtPols ) , len ( detPols ) ] iouMat = np . empty ( outputShape ) gtRectMat = np . zeros ( len ( gtPols ) , np . int8 ) detRectMat = np . zeros ( len ( detPols ) , np . int8 ) for gtNum in range ( len ( gtPols ) ) : for detNum in range ( len ( detPols ) ) : pG = gtPols [ gtNum ] pD = detPols [ detNum ] iouMat [ gtNum , detNum ] = get_intersection_over_union ( pD , pG ) for gtNum in range ( len ( gtPols ) ) : for detNum in range ( len ( detPols ) ) : if gtRectMat [ gtNum ] == 0 and detRectMat [ detNum ] == 0 and gtNum not in gtDontCarePolsNum and detNum not in detDontCarePolsNum : if iouMat [ gtNum , detNum ] > evaluationParams [ 'IOU_CONSTRAINT' ] : gtRectMat [ gtNum ] = 1 detRectMat [ detNum ] = 1 detMatched += 1 pairs . append ( { 'gt' : gtNum , 'det' : detNum } ) detMatchedNums . append ( detNum ) evaluationLog += "Match GT #" + str ( gtNum ) + " with Det #" + str ( detNum ) + "\n" if evaluationParams [ 'CONFIDENCES' ] : for detNum in range ( len ( detPols ) ) : if detNum not in detDontCarePolsNum : # we exclude the don't care detections match = detNum in detMatchedNums arrSampleConfidences . append ( confidencesList [ detNum ] ) arrSampleMatch . append ( match ) arrGlobalConfidences . append ( confidencesList [ detNum ] ) arrGlobalMatches . append ( match ) numGtCare = ( len ( gtPols ) - len ( gtDontCarePolsNum ) ) numDetCare = ( len ( detPols ) - len ( detDontCarePolsNum ) ) if numGtCare == 0 : recall = float ( 1 ) precision = float ( 0 ) if numDetCare > 0 else float ( 1 ) sampleAP = precision else : recall = float ( detMatched ) / numGtCare precision = 0 if numDetCare == 0 else float ( detMatched ) / numDetCare if evaluationParams [ 'CONFIDENCES' ] and evaluationParams [ 'PER_SAMPLE_RESULTS' ] : sampleAP = compute_ap ( arrSampleConfidences , arrSampleMatch , numGtCare ) hmean = 0 if ( precision + recall ) == 0 else 2.0 * precision * recall / ( precision + recall ) matchedSum += detMatched numGlobalCareGt += numGtCare numGlobalCareDet += numDetCare if evaluationParams [ 'PER_SAMPLE_RESULTS' ] : perSampleMetrics [ resFile ] = { 'precision' : precision , 'recall' : recall , 'hmean' : hmean , 'pairs' : pairs , 'AP' : sampleAP , 'iouMat' : [ ] if len ( detPols ) > 100 else iouMat . tolist ( ) , 'gtPolPoints' : gtPolPoints , 'detPolPoints' : detPolPoints , 'gtDontCare' : gtDontCarePolsNum , 'detDontCare' : detDontCarePolsNum , 'evaluationParams' : evaluationParams , 'evaluationLog' : evaluationLog } # Compute MAP and MAR AP = 0 if evaluationParams [ 'CONFIDENCES' ] : AP = compute_ap ( arrGlobalConfidences , arrGlobalMatches , numGlobalCareGt ) methodRecall = 0 if numGlobalCareGt == 0 else float ( matchedSum ) / numGlobalCareGt methodPrecision = 0 if numGlobalCareDet == 0 else float ( matchedSum ) / numGlobalCareDet methodHmean = 0 if methodRecall + methodPrecision == 0 else 2 * methodRecall * methodPrecision / ( methodRecall + methodPrecision ) methodMetrics = { 'precision' : methodPrecision , 'recall' : methodRecall , 'hmean' : methodHmean , 'AP' : AP } resDict = { 'calculated' : True , 'Message' : '' , 'method' : methodMetrics , 'per_sample' : perSampleMetrics } return resDict