(self)
start to fit
start to fit
start to fit
def fit(self):
        """
        start to fit
        """
        LOGGER.info('begin to fit homo decision tree, epoch {}, tree idx {},'
                    'running on distributed backend'.format(self.epoch_idx, self.tree_idx))

        self.init_root_node_and_gh_sum()
        LOGGER.debug('assign samples to root node')
        self.inst2node_idx = self.assign_instance_to_root_node(self.data_bin, 0)

        tree_height = self.max_depth + 1  # non-leaf node height + 1 layer leaf

        for dep in range(tree_height):

            if dep + 1 == tree_height:

                for node in self.cur_layer_node:
                    node.is_leaf = True
                    self.tree_node.append(node)

                rest_sample_leaf_pos = self.inst2node_idx.mapValues(lambda x: x[1])
                if self.sample_leaf_pos is None:
                    self.sample_leaf_pos = rest_sample_leaf_pos
                else:
                    self.sample_leaf_pos = self.sample_leaf_pos.union(rest_sample_leaf_pos)
                # stop fitting
                break

            LOGGER.debug('start to fit layer {}'.format(dep))

            table_with_assignment = self.update_instances_node_positions()

            # send current layer node number:
            self.sync_cur_layer_node_num(len(self.cur_layer_node), suffix=(dep, self.epoch_idx, self.tree_idx))

            split_info, agg_histograms = [], []
            for batch_id, i in enumerate(range(0, len(self.cur_layer_node), self.max_split_nodes)):
                cur_to_split = self.cur_layer_node[i:i+self.max_split_nodes]

                node_map = self.get_node_map(nodes=cur_to_split)
                LOGGER.debug('node map is {}'.format(node_map))
                LOGGER.debug('computing histogram for batch{} at depth{}'.format(batch_id, dep))
                local_histogram = self.get_left_node_local_histogram(
                    cur_nodes=cur_to_split,
                    tree=self.tree_node,
                    g_h=self.g_h,
                    table_with_assign=table_with_assignment,
                    split_points=self.bin_split_points,
                    sparse_point=self.bin_sparse_points,
                    valid_feature=self.valid_features
                )

                LOGGER.debug('federated finding best splits for batch{} at layer {}'.format(batch_id, dep))
                self.sync_local_node_histogram(local_histogram, suffix=(batch_id, dep, self.epoch_idx, self.tree_idx))

                agg_histograms += local_histogram

            split_info = self.sync_best_splits(suffix=(dep, self.epoch_idx))
            LOGGER.debug('got best splits from arbiter')

            new_layer_node = self.update_tree(self.cur_layer_node, split_info)
            self.cur_layer_node = new_layer_node

            self.inst2node_idx, leaf_val = self.assign_instances_to_new_node(table_with_assignment, self.tree_node)
            # record leaf val
            if self.sample_leaf_pos is None:
                self.sample_leaf_pos = leaf_val
            else:
                self.sample_leaf_pos = self.sample_leaf_pos.union(leaf_val)

            LOGGER.debug('assigning instance to new nodes done')

        self.convert_bin_to_real()
        self.sample_weights_post_process()
        LOGGER.debug('fitting tree done')
def fit ( self ) : LOGGER . info ( 'begin to fit homo decision tree, epoch {}, tree idx {},' 'running on distributed backend' . format ( self . epoch_idx , self . tree_idx ) ) self . init_root_node_and_gh_sum ( ) LOGGER . debug ( 'assign samples to root node' ) self . inst2node_idx = self . assign_instance_to_root_node ( self . data_bin , 0 ) tree_height = self . max_depth + 1 # non-leaf node height + 1 layer leaf for dep in range ( tree_height ) : if dep + 1 == tree_height : for node in self . cur_layer_node : node . is_leaf = True self . tree_node . append ( node ) rest_sample_leaf_pos = self . inst2node_idx . mapValues ( lambda x : x [ 1 ] ) if self . sample_leaf_pos is None : self . sample_leaf_pos = rest_sample_leaf_pos else : self . sample_leaf_pos = self . sample_leaf_pos . union ( rest_sample_leaf_pos ) # stop fitting break LOGGER . debug ( 'start to fit layer {}' . format ( dep ) ) table_with_assignment = self . update_instances_node_positions ( ) # send current layer node number: self . sync_cur_layer_node_num ( len ( self . cur_layer_node ) , suffix = ( dep , self . epoch_idx , self . tree_idx ) ) split_info , agg_histograms = [ ] , [ ] for batch_id , i in enumerate ( range ( 0 , len ( self . cur_layer_node ) , self . max_split_nodes ) ) : cur_to_split = self . cur_layer_node [ i : i + self . max_split_nodes ] node_map = self . get_node_map ( nodes = cur_to_split ) LOGGER . debug ( 'node map is {}' . format ( node_map ) ) LOGGER . debug ( 'computing histogram for batch{} at depth{}' . format ( batch_id , dep ) ) local_histogram = self . get_left_node_local_histogram ( cur_nodes = cur_to_split , tree = self . tree_node , g_h = self . g_h , table_with_assign = table_with_assignment , split_points = self . bin_split_points , sparse_point = self . bin_sparse_points , valid_feature = self . valid_features ) LOGGER . debug ( 'federated finding best splits for batch{} at layer {}' . format ( batch_id , dep ) ) self . sync_local_node_histogram ( local_histogram , suffix = ( batch_id , dep , self . epoch_idx , self . tree_idx ) ) agg_histograms += local_histogram split_info = self . sync_best_splits ( suffix = ( dep , self . epoch_idx ) ) LOGGER . debug ( 'got best splits from arbiter' ) new_layer_node = self . update_tree ( self . cur_layer_node , split_info ) self . cur_layer_node = new_layer_node self . inst2node_idx , leaf_val = self . assign_instances_to_new_node ( table_with_assignment , self . tree_node ) # record leaf val if self . sample_leaf_pos is None : self . sample_leaf_pos = leaf_val else : self . sample_leaf_pos = self . sample_leaf_pos . union ( leaf_val ) LOGGER . debug ( 'assigning instance to new nodes done' ) self . convert_bin_to_real ( ) self . sample_weights_post_process ( ) LOGGER . debug ( 'fitting tree done' )