(self, X, y=None, serialize=True)
return return_data
[summary]

        Parameters
        ----------
        X : [type]
            [description]
        y : [type], optional
            by default None
        Returns
        -------
        [type]
            [description]
[summary]
[ summary ]
def transform_with_batches(self, X, y=None, serialize=True):
        """[summary]

        Parameters
        ----------
        X : [type]
            [description]
        y : [type], optional
            by default None
        Returns
        -------
        [type]
            [description]
        """
        outputs = {}
        serialization_out_dir: Text = ""
        if serialize:
            serialization_out_dir = os.path.join(
                os.path.dirname(__file__), "temp_serialized"
            )
            if not os.path.exists(serialization_out_dir):
                os.mkdir(serialization_out_dir)

        sink_nodes = self._find_sink_nodes()
        sink_node = sink_nodes[0]
        operator_idx = 0
        inputs: Any
        output = None

        for batch_idx, batch_data in enumerate(
            X
        ):  # batching_transformer will output only one obj
            if isinstance(batch_data, Tuple):
                batch_X, batch_y = batch_data
            else:
                batch_X = batch_data
                batch_y = None

            for operator in self._steps:
                preds = self._preds[operator]
                if len(preds) == 0:
                    inputs = batch_X
                else:
                    inputs = [
                        outputs[pred][0]
                        if isinstance(outputs[pred], tuple)
                        else outputs[pred]
                        for pred in preds
                    ]
                if len(inputs) == 1:
                    inputs = inputs[0]
                trained = operator
                if trained.is_transformer():
                    batch_output = trained.transform(inputs, batch_y)
                else:
                    if trained in sink_nodes:
                        batch_output = trained._predict(
                            X=inputs
                        )  # We don't support y for predict yet as there is no compelling case
                    else:
                        # This is ok because trainable pipelines steps
                        # must only be individual operators
                        if trained.has_method("predict_proba"):  # type: ignore
                            batch_output = trained.predict_proba(X=inputs)
                        elif trained.has_method("decision_function"):  # type: ignore
                            batch_output = trained.decision_function(X=inputs)
                        else:
                            batch_output = trained._predict(X=inputs)
                if trained == sink_node:
                    if isinstance(batch_output, tuple):
                        output = lale.helpers.append_batch(
                            output, (batch_output[0], batch_output[1])
                        )
                    else:
                        output = lale.helpers.append_batch(output, batch_output)
                outputs[operator] = batch_output
                operator_idx += 1

            #     if serialize:
            #         output = lale.helpers.write_batch_output_to_file(
            #             output,
            #             os.path.join(
            #                 serialization_out_dir,
            #                 "fit_with_batches" + str(operator_idx) + ".hdf5",
            #             ),
            #             len(inputs.dataset),
            #             batch_idx,
            #             batch_X,
            #             batch_y,
            #             batch_out_X,
            #             batch_out_y,
            #         )
            #     else:
            #         if batch_out_y is not None:
            #             output = lale.helpers.append_batch(
            #                 output, (batch_output, batch_out_y)
            #             )
            #         else:
            #             output = lale.helpers.append_batch(output, batch_output)
            # if serialize:
            #     output.close()  # type: ignore
            #     output = lale.helpers.create_data_loader(
            #         os.path.join(
            #             serialization_out_dir,
            #             "fit_with_batches" + str(operator_idx) + ".hdf5",
            #         ),
            #         batch_size=inputs.batch_size,
            #     )
            # else:
            #     if isinstance(output, tuple):
            #         output = lale.helpers.create_data_loader(
            #             X=output[0], y=output[1], batch_size=inputs.batch_size
            #         )
            #     else:
            #         output = lale.helpers.create_data_loader(
            #             X=output, y=None, batch_size=inputs.batch_size
            #         )
            # outputs[operator] = output
            # operator_idx += 1

        return_data = output  # outputs[self._steps[-1]]#.dataset.get_data()
        # if serialize:
        #     shutil.rmtree(serialization_out_dir)

        return return_data
def transform_with_batches ( self , X , y = None , serialize = True ) : outputs = { } serialization_out_dir : Text = "" if serialize : serialization_out_dir = os . path . join ( os . path . dirname ( __file__ ) , "temp_serialized" ) if not os . path . exists ( serialization_out_dir ) : os . mkdir ( serialization_out_dir ) sink_nodes = self . _find_sink_nodes ( ) sink_node = sink_nodes [ 0 ] operator_idx = 0 inputs : Any output = None for batch_idx , batch_data in enumerate ( X ) : # batching_transformer will output only one obj if isinstance ( batch_data , Tuple ) : batch_X , batch_y = batch_data else : batch_X = batch_data batch_y = None for operator in self . _steps : preds = self . _preds [ operator ] if len ( preds ) == 0 : inputs = batch_X else : inputs = [ outputs [ pred ] [ 0 ] if isinstance ( outputs [ pred ] , tuple ) else outputs [ pred ] for pred in preds ] if len ( inputs ) == 1 : inputs = inputs [ 0 ] trained = operator if trained . is_transformer ( ) : batch_output = trained . transform ( inputs , batch_y ) else : if trained in sink_nodes : batch_output = trained . _predict ( X = inputs ) # We don't support y for predict yet as there is no compelling case else : # This is ok because trainable pipelines steps # must only be individual operators if trained . has_method ( "predict_proba" ) : # type: ignore batch_output = trained . predict_proba ( X = inputs ) elif trained . has_method ( "decision_function" ) : # type: ignore batch_output = trained . decision_function ( X = inputs ) else : batch_output = trained . _predict ( X = inputs ) if trained == sink_node : if isinstance ( batch_output , tuple ) : output = lale . helpers . append_batch ( output , ( batch_output [ 0 ] , batch_output [ 1 ] ) ) else : output = lale . helpers . append_batch ( output , batch_output ) outputs [ operator ] = batch_output operator_idx += 1 #     if serialize: #         output = lale.helpers.write_batch_output_to_file( #             output, #             os.path.join( #                 serialization_out_dir, #                 "fit_with_batches" + str(operator_idx) + ".hdf5", #             ), #             len(inputs.dataset), #             batch_idx, #             batch_X, #             batch_y, #             batch_out_X, #             batch_out_y, #         ) #     else: #         if batch_out_y is not None: #             output = lale.helpers.append_batch( #                 output, (batch_output, batch_out_y) #             ) #         else: #             output = lale.helpers.append_batch(output, batch_output) # if serialize: #     output.close()  # type: ignore #     output = lale.helpers.create_data_loader( #         os.path.join( #             serialization_out_dir, #             "fit_with_batches" + str(operator_idx) + ".hdf5", #         ), #         batch_size=inputs.batch_size, #     ) # else: #     if isinstance(output, tuple): #         output = lale.helpers.create_data_loader( #             X=output[0], y=output[1], batch_size=inputs.batch_size #         ) #     else: #         output = lale.helpers.create_data_loader( #             X=output, y=None, batch_size=inputs.batch_size #         ) # outputs[operator] = output # operator_idx += 1 return_data = output # outputs[self._steps[-1]]#.dataset.get_data() # if serialize: #     shutil.rmtree(serialization_out_dir) return return_data