(self)
return domain_df
Generates the domain for each cell in the active attributes as well
        as assigns a random variable ID (_vid_) for cells that have
        a domain of size >= 2.

        See get_domain_cell for how the domain is generated from co-occurrence
        and correlated attributes.

        If no values can be found from correlated attributes, return a random
        sample of domain values.

        :return: DataFrame with columns
            _tid_: entity/tuple ID
            _cid_: cell ID (one for every cell in the raw data in active attributes)
            _vid_: random variable ID (one for every cell with a domain of at least size 2)
            attribute: attribute name
            domain: ||| separated string of domain values
            domain_size: length of domain
            init_value: initial value for this cell
            init_value_idx: domain index of init_value
            fixed: 1 if a random sample was taken since no correlated attributes/top K values
Generates the domain for each cell in the active attributes as well
        as assigns a random variable ID (_vid_) for cells that have
        a domain of size >= 2.
Generates the domain for each cell in the active attributes as well as assigns a random variable ID ( _vid_ ) for cells that have a domain of size > = 2 .
def generate_domain(self):
        """
        Generates the domain for each cell in the active attributes as well
        as assigns a random variable ID (_vid_) for cells that have
        a domain of size >= 2.

        See get_domain_cell for how the domain is generated from co-occurrence
        and correlated attributes.

        If no values can be found from correlated attributes, return a random
        sample of domain values.

        :return: DataFrame with columns
            _tid_: entity/tuple ID
            _cid_: cell ID (one for every cell in the raw data in active attributes)
            _vid_: random variable ID (one for every cell with a domain of at least size 2)
            attribute: attribute name
            domain: ||| separated string of domain values
            domain_size: length of domain
            init_value: initial value for this cell
            init_value_idx: domain index of init_value
            fixed: 1 if a random sample was taken since no correlated attributes/top K values
        """

        if not self.setup_complete:
            raise Exception(
                "Call <setup_attributes> to setup active attributes. Error detection should be performed before setup.")

        logging.debug('generating initial set of un-pruned domain values...')
        tic = time.clock()
        # Iterate over dataset rows.
        cells = []
        vid = 0
        records = self.ds.get_raw_data().to_records()
        self.all_attrs = list(records.dtype.names)
        for row in tqdm(list(records)):
            tid = row['_tid_']
            for attr in self.active_attributes:
                init_value, init_value_idx, dom = self.get_domain_cell(attr, row)
                # We will use an estimator model for additional weak labelling
                # below, which requires an initial pruned domain first.
                # Weak labels will be trained on the init values.
                cid = self.ds.get_cell_id(tid, attr)

                # Originally, all cells have a NOT_SET status to be considered
                # in weak labelling.
                cell_status = CellStatus.NOT_SET.value

                if len(dom) <= 1:
                    # Initial  value is NULL and we cannot come up with
                    # a domain; a random domain probably won't help us so
                    # completely ignore this cell and continue.
                    # Note if len(dom) == 1, then we generated a single correct
                    # value (since NULL is not included in the domain).
                    # This would be a "SINGLE_VALUE" example and we'd still
                    # like to generate a random domain for it.
                    if init_value == NULL_REPR and len(dom) == 0:
                        continue

                    # Not enough domain values, we need to get some random
                    # values (other than 'init_value') for training. However,
                    # this might still get us zero domain values.
                    rand_dom_values = self.get_random_domain(attr, init_value)

                    # rand_dom_values might still be empty. In this case,
                    # there are no other possible values for this cell. There
                    # is not point to use this cell for training and there is no
                    # point to run inference on it since we cannot even generate
                    # a random domain. Therefore, we just ignore it from the
                    # final tensor.
                    if len(rand_dom_values) == 0:
                        continue

                    # Otherwise, just add the random domain values to the domain
                    # and set the cell status accordingly.
                    dom.extend(rand_dom_values)

                    # Set the cell status that this is a single value and was
                    # randomly assigned other values in the domain. These will
                    # not be modified by the estimator.
                    cell_status = CellStatus.SINGLE_VALUE.value

                cells.append({"_tid_": tid,
                              "attribute": attr,
                              "_cid_": cid,
                              "_vid_": vid,
                              "domain": "|||".join(dom),
                              "domain_size": len(dom),
                              "init_value": init_value,
                              "init_index": init_value_idx,
                              "weak_label": init_value,
                              "weak_label_idx": init_value_idx,
                              "fixed": cell_status})
                vid += 1
        domain_df = pd.DataFrame(data=cells).sort_values('_vid_')
        logging.debug('DONE generating initial set of domain values in %.2f', time.clock() - tic)

        # Skip estimator model since we do not require any weak labelling or domain
        # pruning based on posterior probabilities.
        if self.env['weak_label_thresh'] == 1 and self.env['domain_thresh_2'] == 0:
            return domain_df

        # Run pruned domain values from correlated attributes above through
        # posterior model for a naive probability estimation.
        logging.debug('training posterior model for estimating domain value probabilities...')
        tic = time.clock()
        estimator = NaiveBayes(self.env, self.ds, domain_df, self.correlations)
        logging.debug('DONE training posterior model in %.2fs', time.clock() - tic)

        # Predict probabilities for all pruned domain values.
        logging.debug('predicting domain value probabilities from posterior model...')
        tic = time.clock()
        preds_by_cell = estimator.predict_pp_batch()
        logging.debug('DONE predictions in %.2f secs, re-constructing cell domain...', time.clock() - tic)

        logging.debug('re-assembling final cell domain table...')
        tic = time.clock()
        # iterate through raw/current data and generate posterior probabilities for
        # weak labelling
        num_weak_labels = 0
        updated_domain_df = []
        for preds, row in tqdm(zip(preds_by_cell, domain_df.to_records())):
            # Do not re-label single valued cells.
            if row['fixed'] == CellStatus.SINGLE_VALUE.value:
                updated_domain_df.append(row)
                continue

            # prune domain if any of the values are above our domain_thresh_2
            preds = [[val, proba] for val, proba in preds if proba >= self.domain_thresh_2] or preds

            # cap the maximum # of domain values to self.max_domain based on probabilities.
            domain_values = [val for val, proba in sorted(preds, key=lambda pred: pred[1], reverse=True)[:self.max_domain]]

            # ensure the initial value is included even if its probability is low.
            if row['init_value'] not in domain_values and row['init_value'] != NULL_REPR:
                domain_values.append(row['init_value'])
            domain_values = sorted(domain_values)
            # update our memoized domain values for this row again
            row['domain'] = '|||'.join(domain_values)
            row['domain_size'] = len(domain_values)
            # update init index based on new domain
            if row['init_value'] in domain_values:
                row['init_index'] = domain_values.index(row['init_value'])
            # update weak label index based on new domain
            if row['weak_label'] != NULL_REPR:
                row['weak_label_idx'] = domain_values.index(row['weak_label'])

            weak_label, weak_label_prob = max(preds, key=lambda pred: pred[1])

            # Assign weak label if it is not the same as init AND domain value
            # exceeds our weak label threshold.
            if weak_label != row['init_value'] and weak_label_prob >= self.weak_label_thresh:
                num_weak_labels += 1

                weak_label_idx = domain_values.index(weak_label)
                row['weak_label'] = weak_label
                row['weak_label_idx'] = weak_label_idx
                row['fixed'] = CellStatus.WEAK_LABEL.value

            updated_domain_df.append(row)

        # update our cell domain df with our new updated domain
        domain_df = pd.DataFrame.from_records(updated_domain_df, columns=updated_domain_df[0].dtype.names).drop('index', axis=1).sort_values('_vid_')
        logging.debug('DONE assembling cell domain table in %.2fs', time.clock() - tic)

        logging.info('number of (additional) weak labels assigned from posterior model: %d', num_weak_labels)

        logging.debug('DONE generating domain and weak labels')
        return domain_df
def generate_domain ( self ) : if not self . setup_complete : raise Exception ( "Call <setup_attributes> to setup active attributes. Error detection should be performed before setup." ) logging . debug ( 'generating initial set of un-pruned domain values...' ) tic = time . clock ( ) # Iterate over dataset rows. cells = [ ] vid = 0 records = self . ds . get_raw_data ( ) . to_records ( ) self . all_attrs = list ( records . dtype . names ) for row in tqdm ( list ( records ) ) : tid = row [ '_tid_' ] for attr in self . active_attributes : init_value , init_value_idx , dom = self . get_domain_cell ( attr , row ) # We will use an estimator model for additional weak labelling # below, which requires an initial pruned domain first. # Weak labels will be trained on the init values. cid = self . ds . get_cell_id ( tid , attr ) # Originally, all cells have a NOT_SET status to be considered # in weak labelling. cell_status = CellStatus . NOT_SET . value if len ( dom ) <= 1 : # Initial  value is NULL and we cannot come up with # a domain; a random domain probably won't help us so # completely ignore this cell and continue. # Note if len(dom) == 1, then we generated a single correct # value (since NULL is not included in the domain). # This would be a "SINGLE_VALUE" example and we'd still # like to generate a random domain for it. if init_value == NULL_REPR and len ( dom ) == 0 : continue # Not enough domain values, we need to get some random # values (other than 'init_value') for training. However, # this might still get us zero domain values. rand_dom_values = self . get_random_domain ( attr , init_value ) # rand_dom_values might still be empty. In this case, # there are no other possible values for this cell. There # is not point to use this cell for training and there is no # point to run inference on it since we cannot even generate # a random domain. Therefore, we just ignore it from the # final tensor. if len ( rand_dom_values ) == 0 : continue # Otherwise, just add the random domain values to the domain # and set the cell status accordingly. dom . extend ( rand_dom_values ) # Set the cell status that this is a single value and was # randomly assigned other values in the domain. These will # not be modified by the estimator. cell_status = CellStatus . SINGLE_VALUE . value cells . append ( { "_tid_" : tid , "attribute" : attr , "_cid_" : cid , "_vid_" : vid , "domain" : "|||" . join ( dom ) , "domain_size" : len ( dom ) , "init_value" : init_value , "init_index" : init_value_idx , "weak_label" : init_value , "weak_label_idx" : init_value_idx , "fixed" : cell_status } ) vid += 1 domain_df = pd . DataFrame ( data = cells ) . sort_values ( '_vid_' ) logging . debug ( 'DONE generating initial set of domain values in %.2f' , time . clock ( ) - tic ) # Skip estimator model since we do not require any weak labelling or domain # pruning based on posterior probabilities. if self . env [ 'weak_label_thresh' ] == 1 and self . env [ 'domain_thresh_2' ] == 0 : return domain_df # Run pruned domain values from correlated attributes above through # posterior model for a naive probability estimation. logging . debug ( 'training posterior model for estimating domain value probabilities...' ) tic = time . clock ( ) estimator = NaiveBayes ( self . env , self . ds , domain_df , self . correlations ) logging . debug ( 'DONE training posterior model in %.2fs' , time . clock ( ) - tic ) # Predict probabilities for all pruned domain values. logging . debug ( 'predicting domain value probabilities from posterior model...' ) tic = time . clock ( ) preds_by_cell = estimator . predict_pp_batch ( ) logging . debug ( 'DONE predictions in %.2f secs, re-constructing cell domain...' , time . clock ( ) - tic ) logging . debug ( 're-assembling final cell domain table...' ) tic = time . clock ( ) # iterate through raw/current data and generate posterior probabilities for # weak labelling num_weak_labels = 0 updated_domain_df = [ ] for preds , row in tqdm ( zip ( preds_by_cell , domain_df . to_records ( ) ) ) : # Do not re-label single valued cells. if row [ 'fixed' ] == CellStatus . SINGLE_VALUE . value : updated_domain_df . append ( row ) continue # prune domain if any of the values are above our domain_thresh_2 preds = [ [ val , proba ] for val , proba in preds if proba >= self . domain_thresh_2 ] or preds # cap the maximum # of domain values to self.max_domain based on probabilities. domain_values = [ val for val , proba in sorted ( preds , key = lambda pred : pred [ 1 ] , reverse = True ) [ : self . max_domain ] ] # ensure the initial value is included even if its probability is low. if row [ 'init_value' ] not in domain_values and row [ 'init_value' ] != NULL_REPR : domain_values . append ( row [ 'init_value' ] ) domain_values = sorted ( domain_values ) # update our memoized domain values for this row again row [ 'domain' ] = '|||' . join ( domain_values ) row [ 'domain_size' ] = len ( domain_values ) # update init index based on new domain if row [ 'init_value' ] in domain_values : row [ 'init_index' ] = domain_values . index ( row [ 'init_value' ] ) # update weak label index based on new domain if row [ 'weak_label' ] != NULL_REPR : row [ 'weak_label_idx' ] = domain_values . index ( row [ 'weak_label' ] ) weak_label , weak_label_prob = max ( preds , key = lambda pred : pred [ 1 ] ) # Assign weak label if it is not the same as init AND domain value # exceeds our weak label threshold. if weak_label != row [ 'init_value' ] and weak_label_prob >= self . weak_label_thresh : num_weak_labels += 1 weak_label_idx = domain_values . index ( weak_label ) row [ 'weak_label' ] = weak_label row [ 'weak_label_idx' ] = weak_label_idx row [ 'fixed' ] = CellStatus . WEAK_LABEL . value updated_domain_df . append ( row ) # update our cell domain df with our new updated domain domain_df = pd . DataFrame . from_records ( updated_domain_df , columns = updated_domain_df [ 0 ] . dtype . names ) . drop ( 'index' , axis = 1 ) . sort_values ( '_vid_' ) logging . debug ( 'DONE assembling cell domain table in %.2fs' , time . clock ( ) - tic ) logging . info ( 'number of (additional) weak labels assigned from posterior model: %d' , num_weak_labels ) logging . debug ( 'DONE generating domain and weak labels' ) return domain_df