(gold, predicted_sets)
return prec, rec, f1
Compute retrieval precision, recall and f-score for the case when each predicted entity has alternative labels.
    Note that it doesn't take into account the order
    and repeating elements.

    :param gold: the set of gold retrieved elements.
    :param predicted_sets: the set of predicted elements.
    :return: a triple of precision, recall and f-score
    >>> retrieval_prec_rec_f1_with_altlabels(['Star Wars', 'Black Swan', 'Thor', 'Leon'], [['thor','Thor','God of thunder'], ['Avengers', 'Defenders'], ['Iron Man']])
    (0.3333333333333333, 0.25, 0.28571428571428575)
    >>> retrieval_prec_rec_f1_with_altlabels(['Star Wars', 'Black Swan', 'Thor', 'Leon'], [])
    (1.0, 0.0, 0.0)
    >>> retrieval_prec_rec_f1_with_altlabels(['Star Wars', 'Black Swan', 'Thor', 'Leon'], [[],[]])
    (1.0, 0.0, 0.0)
    >>> retrieval_prec_rec_f1_with_altlabels(['Leon'], [['Black Swan'],['Leon']])
    (0.5, 1.0, 0.6666666666666666)
    >>> retrieval_prec_rec_f1_with_altlabels([], [['Black Swan'],['Leon']])
    (0.0, 1.0, 0.0)
    >>> retrieval_prec_rec_f1_with_altlabels(['Thor'], [['Thor'], ['Black Swan'],['Leon']])
    (0.3333333333333333, 1.0, 0.5)
    >>> retrieval_prec_rec_f1_with_altlabels(['Thor', 'Brian'], [['Brian'], ['Black Swan'],['Leon']])
    (0.3333333333333333, 0.5, 0.4)
Compute retrieval precision, recall and f-score for the case when each predicted entity has alternative labels.
    Note that it doesn't take into account the order
    and repeating elements.
Compute retrieval precision recall and f - score for the case when each predicted entity has alternative labels . Note that it doesn t take into account the order and repeating elements .
def retrieval_prec_rec_f1_with_altlabels(gold, predicted_sets):
    """
    Compute retrieval precision, recall and f-score for the case when each predicted entity has alternative labels.
    Note that it doesn't take into account the order
    and repeating elements.

    :param gold: the set of gold retrieved elements.
    :param predicted_sets: the set of predicted elements.
    :return: a triple of precision, recall and f-score
    >>> retrieval_prec_rec_f1_with_altlabels(['Star Wars', 'Black Swan', 'Thor', 'Leon'], [['thor','Thor','God of thunder'], ['Avengers', 'Defenders'], ['Iron Man']])
    (0.3333333333333333, 0.25, 0.28571428571428575)
    >>> retrieval_prec_rec_f1_with_altlabels(['Star Wars', 'Black Swan', 'Thor', 'Leon'], [])
    (1.0, 0.0, 0.0)
    >>> retrieval_prec_rec_f1_with_altlabels(['Star Wars', 'Black Swan', 'Thor', 'Leon'], [[],[]])
    (1.0, 0.0, 0.0)
    >>> retrieval_prec_rec_f1_with_altlabels(['Leon'], [['Black Swan'],['Leon']])
    (0.5, 1.0, 0.6666666666666666)
    >>> retrieval_prec_rec_f1_with_altlabels([], [['Black Swan'],['Leon']])
    (0.0, 1.0, 0.0)
    >>> retrieval_prec_rec_f1_with_altlabels(['Thor'], [['Thor'], ['Black Swan'],['Leon']])
    (0.3333333333333333, 1.0, 0.5)
    >>> retrieval_prec_rec_f1_with_altlabels(['Thor', 'Brian'], [['Brian'], ['Black Swan'],['Leon']])
    (0.3333333333333333, 0.5, 0.4)
    """
    tp = retrieval_tp_with_altlabels(gold, predicted_sets)
    prec = tp / len(predicted_sets) if len(predicted_sets) and sum(len(s) for s in predicted_sets) > 0 else 1.0
    rec = tp / len(gold) if len(gold) > 0 else 1.0
    f1 = 0.0
    if (rec+prec) > 0:
        f1 = 2.0 * prec * rec / (prec + rec)

    return prec, rec, f1
def retrieval_prec_rec_f1_with_altlabels ( gold , predicted_sets ) : tp = retrieval_tp_with_altlabels ( gold , predicted_sets ) prec = tp / len ( predicted_sets ) if len ( predicted_sets ) and sum ( len ( s ) for s in predicted_sets ) > 0 else 1.0 rec = tp / len ( gold ) if len ( gold ) > 0 else 1.0 f1 = 0.0 if ( rec + prec ) > 0 : f1 = 2.0 * prec * rec / ( prec + rec ) return prec , rec , f1