(self, data_bin, grad_and_hess,
                            bin_split_points, bin_sparse_points,
                            valid_features=None,
                            node_map=None,
                            use_missing=False,
                            zero_as_missing=False,
                            parent_node_id_map=None,
                            sibling_node_id_map=None,
                            ret="tensor")
This is the old interface for histogram computation

        data_bin: data after binning with node positions
        grad_and_hess: g/h for each sample
        bin_split_points: split points
        bin_sparse_points: sparse points
        node_map: node id to node index
        use_missing: enable use missing
        zero_as_missing: enable zero as missing
        parent_node_id_map: map current node_id to its parent id, this para is for hist sub
        sibling_node_id_map: map current node_id to its sibling id, this para is for hist sub
        ret: return type, if 'tb', return histograms stored in Table
This is the old interface for histogram computation
This is the old interface for histogram computation
def calculate_histogram(self, data_bin, grad_and_hess,
                            bin_split_points, bin_sparse_points,
                            valid_features=None,
                            node_map=None,
                            use_missing=False,
                            zero_as_missing=False,
                            parent_node_id_map=None,
                            sibling_node_id_map=None,
                            ret="tensor"):

        """
        This is the old interface for histogram computation

        data_bin: data after binning with node positions
        grad_and_hess: g/h for each sample
        bin_split_points: split points
        bin_sparse_points: sparse points
        node_map: node id to node index
        use_missing: enable use missing
        zero_as_missing: enable zero as missing
        parent_node_id_map: map current node_id to its parent id, this para is for hist sub
        sibling_node_id_map: map current node_id to its sibling id, this para is for hist sub
        ret: return type, if 'tb', return histograms stored in Table
        """

        LOGGER.debug("bin_shape is {}, node num is {}".format(bin_split_points.shape, len(node_map)))

        # reformat, now format is: key, ((data_instance, node position), (g, h))
        batch_histogram_intermediate_rs = data_bin.join(grad_and_hess, lambda data_inst, g_h: (data_inst, g_h))

        if batch_histogram_intermediate_rs.count() == 0: # if input sample number is 0, return empty histograms

            node_histograms = FeatureHistogram._generate_histogram_template(node_map, bin_split_points, valid_features,
                                                                            1 if use_missing else 0)
            hist_list = FeatureHistogram._generate_histogram_key_value_list(node_histograms, node_map, bin_split_points,
                                                                            parent_node_id_map=parent_node_id_map,
                                                                            sibling_node_id_map=sibling_node_id_map)

            if ret == 'tensor':
                feature_num = bin_split_points.shape[0]
                return FeatureHistogram._recombine_histograms(hist_list, node_map, feature_num)
            else:
                histograms_table = session.parallelize(hist_list, partition=data_bin.partitions, include_key=True)
                return FeatureHistogram._construct_table(histograms_table)

        else:  # compute histograms

            batch_histogram_cal = functools.partial(
                FeatureHistogram._batch_calculate_histogram,
                bin_split_points=bin_split_points, bin_sparse_points=bin_sparse_points,
                valid_features=valid_features, node_map=node_map,
                use_missing=use_missing, zero_as_missing=zero_as_missing,
                parent_nid_map=parent_node_id_map,
                sibling_node_id_map=sibling_node_id_map,
                stable_reduce=self.stable_reduce
            )

            agg_func = self._stable_hist_aggregate if self.stable_reduce else self._hist_aggregate
            histograms_table = batch_histogram_intermediate_rs.mapReducePartitions(batch_histogram_cal, agg_func)
            if self.stable_reduce:
                histograms_table = histograms_table.mapValues(self._stable_hist_reduce)

            if ret == "tensor":
                feature_num = bin_split_points.shape[0]
                histogram_list = list(histograms_table.collect())
                rs = FeatureHistogram._recombine_histograms(histogram_list, node_map, feature_num)
                return rs
            else:
                return FeatureHistogram._construct_table(histograms_table)
def calculate_histogram ( self , data_bin , grad_and_hess , bin_split_points , bin_sparse_points , valid_features = None , node_map = None , use_missing = False , zero_as_missing = False , parent_node_id_map = None , sibling_node_id_map = None , ret = "tensor" ) : LOGGER . debug ( "bin_shape is {}, node num is {}" . format ( bin_split_points . shape , len ( node_map ) ) ) # reformat, now format is: key, ((data_instance, node position), (g, h)) batch_histogram_intermediate_rs = data_bin . join ( grad_and_hess , lambda data_inst , g_h : ( data_inst , g_h ) ) if batch_histogram_intermediate_rs . count ( ) == 0 : # if input sample number is 0, return empty histograms node_histograms = FeatureHistogram . _generate_histogram_template ( node_map , bin_split_points , valid_features , 1 if use_missing else 0 ) hist_list = FeatureHistogram . _generate_histogram_key_value_list ( node_histograms , node_map , bin_split_points , parent_node_id_map = parent_node_id_map , sibling_node_id_map = sibling_node_id_map ) if ret == 'tensor' : feature_num = bin_split_points . shape [ 0 ] return FeatureHistogram . _recombine_histograms ( hist_list , node_map , feature_num ) else : histograms_table = session . parallelize ( hist_list , partition = data_bin . partitions , include_key = True ) return FeatureHistogram . _construct_table ( histograms_table ) else : # compute histograms batch_histogram_cal = functools . partial ( FeatureHistogram . _batch_calculate_histogram , bin_split_points = bin_split_points , bin_sparse_points = bin_sparse_points , valid_features = valid_features , node_map = node_map , use_missing = use_missing , zero_as_missing = zero_as_missing , parent_nid_map = parent_node_id_map , sibling_node_id_map = sibling_node_id_map , stable_reduce = self . stable_reduce ) agg_func = self . _stable_hist_aggregate if self . stable_reduce else self . _hist_aggregate histograms_table = batch_histogram_intermediate_rs . mapReducePartitions ( batch_histogram_cal , agg_func ) if self . stable_reduce : histograms_table = histograms_table . mapValues ( self . _stable_hist_reduce ) if ret == "tensor" : feature_num = bin_split_points . shape [ 0 ] histogram_list = list ( histograms_table . collect ( ) ) rs = FeatureHistogram . _recombine_histograms ( histogram_list , node_map , feature_num ) return rs else : return FeatureHistogram . _construct_table ( histograms_table )