(self, data_inst)
override parent function
override parent function
override parent function
def data_preporcess(self, data_inst):
        """
        override parent function
        """
        need_transform_to_sparse = self.backend == consts.DISTRIBUTED_BACKEND or\
                                   (self.backend == consts.MEMORY_BACKEND and self.use_missing and self.zero_as_missing)

        backup_schema = copy.deepcopy(data_inst.schema)
        if self.backend == consts.MEMORY_BACKEND:
            # memory backend only support dense format input
            data_example = data_inst.take(1)[0][1]
            if type(data_example.features) == SparseVector:
                recover_func = functools.partial(self._sparse_recover, feat_num=len(data_inst.schema['header']))
                data_inst = data_inst.mapValues(recover_func)
                data_inst.schema = backup_schema

        if need_transform_to_sparse:
            data_inst = self.data_alignment(data_inst)
        elif self.use_missing:
            # fill nan
            data_inst = data_inst.mapValues(self._fill_nan)
            data_inst.schema = backup_schema

        self.data_bin, self.bin_split_points, self.bin_sparse_points = self.federated_binning(data_inst)

        if self.backend == consts.MEMORY_BACKEND:

            if self.use_missing and self.zero_as_missing:
                feat_num = len(self.bin_split_points)
                func = functools.partial(self._handle_zero_as_missing, feat_num=feat_num, missing_bin_idx=self.bin_num)
                self.data_bin = self.data_bin.mapValues(func)
            elif self.use_missing:  # use missing only
                missing_bin_index = self.bin_num
                func = functools.partial(self._map_missing_bin, bin_index=missing_bin_index)
                self.data_bin = self.data_bin.mapValues(func)

            self._collect_data_arr(self.data_bin)
def data_preporcess ( self , data_inst ) : need_transform_to_sparse = self . backend == consts . DISTRIBUTED_BACKEND or ( self . backend == consts . MEMORY_BACKEND and self . use_missing and self . zero_as_missing ) backup_schema = copy . deepcopy ( data_inst . schema ) if self . backend == consts . MEMORY_BACKEND : # memory backend only support dense format input data_example = data_inst . take ( 1 ) [ 0 ] [ 1 ] if type ( data_example . features ) == SparseVector : recover_func = functools . partial ( self . _sparse_recover , feat_num = len ( data_inst . schema [ 'header' ] ) ) data_inst = data_inst . mapValues ( recover_func ) data_inst . schema = backup_schema if need_transform_to_sparse : data_inst = self . data_alignment ( data_inst ) elif self . use_missing : # fill nan data_inst = data_inst . mapValues ( self . _fill_nan ) data_inst . schema = backup_schema self . data_bin , self . bin_split_points , self . bin_sparse_points = self . federated_binning ( data_inst ) if self . backend == consts . MEMORY_BACKEND : if self . use_missing and self . zero_as_missing : feat_num = len ( self . bin_split_points ) func = functools . partial ( self . _handle_zero_as_missing , feat_num = feat_num , missing_bin_idx = self . bin_num ) self . data_bin = self . data_bin . mapValues ( func ) elif self . use_missing : # use missing only missing_bin_index = self . bin_num func = functools . partial ( self . _map_missing_bin , bin_index = missing_bin_index ) self . data_bin = self . data_bin . mapValues ( func ) self . _collect_data_arr ( self . data_bin )