(
        self, X, y=None, serialize=True, num_epochs_batching=None
    )
return result
[summary]

        Parameters
        ----------
        X :
            [description]
        y : [type], optional
            For a supervised pipeline, this is an array with the unique class labels
            in the entire dataset, by default None
        Returns
        -------
        [type]
            [description]
[summary]
[ summary ]
def fit_with_batches(
        self, X, y=None, serialize=True, num_epochs_batching=None
    ) -> "TrainedPipeline[TrainedIndividualOp]":
        """[summary]

        Parameters
        ----------
        X :
            [description]
        y : [type], optional
            For a supervised pipeline, this is an array with the unique class labels
            in the entire dataset, by default None
        Returns
        -------
        [type]
            [description]
        """
        estimator_only = True
        concat_features = False
        from lale.lib.lale import ConcatFeatures

        for operator in self._steps[:-1]:
            if not operator.is_frozen_trained():
                estimator_only = False
            if isinstance(operator, ConcatFeatures):  # type:ignore
                concat_features = True
        if not estimator_only and concat_features:
            raise ValueError(
                "A pipeline with ConcatFeatures where the prefix is not frozen trained can not be fitted batch-wise."
            )
        elif estimator_only:
            return self._fit_last_node_with_batches(X, y, num_epochs_batching=None)
        trained_steps: List[TrainedIndividualOp] = []
        outputs: Dict[Operator, Any] = {}
        edges: List[Tuple[TrainableOpType, TrainableOpType]] = self.edges()
        trained_map: Dict[TrainableOpType, TrainedIndividualOp] = {}
        serialization_out_dir: Text = ""
        if serialize:
            serialization_out_dir = os.path.join(
                os.path.dirname(__file__), "temp_serialized"
            )
            if not os.path.exists(serialization_out_dir):
                os.mkdir(serialization_out_dir)

        sink_nodes = self._find_sink_nodes()
        operator_idx = 0
        for operator in self._steps:
            preds = self._preds[operator]
            if len(preds) == 0:
                inputs = [X]
            else:
                inputs = [
                    outputs[pred][0]
                    if isinstance(outputs[pred], tuple)
                    else outputs[pred]
                    for pred in preds
                ]
            trainable = operator
            if len(inputs) == 1:
                inputs = inputs[0]
            trained: Optional[IndividualOp] = None
            num_epochs = 0
            if trainable.has_method("partial_fit"):
                try:
                    num_epochs = trainable._impl_instance().num_epochs
                except AttributeError:
                    if num_epochs_batching is None:
                        warnings.warn(
                            "Operator {} does not have num_epochs and none given to Batching operator, using 1 as a default".format(
                                trainable.name()
                            )
                        )
                        num_epochs = 1
                    else:
                        num_epochs = num_epochs_batching
                assert num_epochs >= 0
            elif trainable.is_frozen_trained():
                pass
            else:
                raise AttributeError(
                    "All operators to be trained with batching need to implement partial_fit. {} doesn't.".format(
                        operator.name()
                    )
                )
            inputs_for_transform: Any = inputs
            if trainable.is_frozen_trained():
                trained = trainable
            else:
                for epoch in range(num_epochs):
                    for _, batch_data in enumerate(
                        inputs
                    ):  # batching_transformer will output only one obj
                        if isinstance(batch_data, tuple):
                            batch_X, batch_y = batch_data
                        elif isinstance(batch_data, list):
                            batch_X = batch_data[0]
                            batch_y = batch_data[1]
                        else:
                            batch_X = batch_data
                            batch_y = None
                        if trainable.is_supervised():
                            try:
                                trained = trainable.partial_fit(
                                    batch_X, batch_y, classes=y
                                )
                            except TypeError:
                                trained = trainable.partial_fit(batch_X, batch_y)
                        else:
                            trained = trainable.partial_fit(batch_X)
            assert trained is not None
            trained = TrainedIndividualOp(
                trained.name(),
                trained._impl,
                trained._schemas,
                None,
                _lale_trained=True,
            )
            trained_map[operator] = trained
            trained_steps.append(trained)

            output = None
            for batch_idx, batch_data in enumerate(
                inputs_for_transform
            ):  # batching_transformer will output only one obj
                if isinstance(batch_data, tuple):
                    batch_X, batch_y = batch_data
                elif isinstance(batch_data, list):
                    batch_X = batch_data[0]
                    batch_y = batch_data[1]
                else:
                    batch_X = batch_data
                    batch_y = None
                if trained.is_transformer():
                    batch_output = trained.transform(batch_X, batch_y)
                else:
                    if trainable in sink_nodes:
                        batch_output = trained._predict(
                            X=batch_X
                        )  # We don't support y for predict yet as there is no compelling case
                    else:
                        # This is ok because trainable pipelines steps
                        # must only be individual operators
                        if trained.has_method("predict_proba"):  # type: ignore
                            batch_output = trained.predict_proba(X=batch_X)
                        elif trained.has_method("decision_function"):  # type: ignore
                            batch_output = trained.decision_function(X=batch_X)
                        else:
                            batch_output = trained._predict(X=batch_X)
                if isinstance(batch_output, tuple):
                    batch_out_X, batch_out_y = batch_output
                else:
                    batch_out_X = batch_output
                    batch_out_y = None
                if serialize:
                    output = lale.helpers.write_batch_output_to_file(
                        output,
                        os.path.join(
                            serialization_out_dir,
                            "fit_with_batches" + str(operator_idx) + ".hdf5",
                        ),
                        len(inputs_for_transform.dataset),
                        batch_idx,
                        batch_X,
                        batch_y,
                        batch_out_X,
                        batch_out_y,
                    )
                else:
                    if batch_out_y is None:
                        output = lale.helpers.append_batch(
                            output, (batch_output, batch_y)
                        )
                    else:
                        output = lale.helpers.append_batch(output, batch_output)
            if serialize:
                output.close()  # type: ignore

                output = lale.helpers.create_data_loader(
                    os.path.join(
                        serialization_out_dir,
                        "fit_with_batches" + str(operator_idx) + ".hdf5",
                    ),
                    batch_size=inputs_for_transform.batch_size,
                )
            else:
                if isinstance(output, tuple):
                    output = lale.helpers.create_data_loader(
                        X=output[0],
                        y=output[1],
                        batch_size=inputs_for_transform.batch_size,
                    )
                else:
                    output = lale.helpers.create_data_loader(
                        X=output, y=None, batch_size=inputs_for_transform.batch_size
                    )
            outputs[operator] = output
            operator_idx += 1

        if serialize:
            shutil.rmtree(serialization_out_dir)
        trained_edges = [(trained_map[x], trained_map[y]) for (x, y) in edges]

        trained_steps2: Any = trained_steps
        result: TrainedPipeline[TrainedIndividualOp] = TrainedPipeline(
            trained_steps2, trained_edges, ordered=True, _lale_trained=True
        )
        self._trained = result
        return result
def fit_with_batches ( self , X , y = None , serialize = True , num_epochs_batching = None ) -> "TrainedPipeline[TrainedIndividualOp]" : estimator_only = True concat_features = False from lale . lib . lale import ConcatFeatures for operator in self . _steps [ : - 1 ] : if not operator . is_frozen_trained ( ) : estimator_only = False if isinstance ( operator , ConcatFeatures ) : # type:ignore concat_features = True if not estimator_only and concat_features : raise ValueError ( "A pipeline with ConcatFeatures where the prefix is not frozen trained can not be fitted batch-wise." ) elif estimator_only : return self . _fit_last_node_with_batches ( X , y , num_epochs_batching = None ) trained_steps : List [ TrainedIndividualOp ] = [ ] outputs : Dict [ Operator , Any ] = { } edges : List [ Tuple [ TrainableOpType , TrainableOpType ] ] = self . edges ( ) trained_map : Dict [ TrainableOpType , TrainedIndividualOp ] = { } serialization_out_dir : Text = "" if serialize : serialization_out_dir = os . path . join ( os . path . dirname ( __file__ ) , "temp_serialized" ) if not os . path . exists ( serialization_out_dir ) : os . mkdir ( serialization_out_dir ) sink_nodes = self . _find_sink_nodes ( ) operator_idx = 0 for operator in self . _steps : preds = self . _preds [ operator ] if len ( preds ) == 0 : inputs = [ X ] else : inputs = [ outputs [ pred ] [ 0 ] if isinstance ( outputs [ pred ] , tuple ) else outputs [ pred ] for pred in preds ] trainable = operator if len ( inputs ) == 1 : inputs = inputs [ 0 ] trained : Optional [ IndividualOp ] = None num_epochs = 0 if trainable . has_method ( "partial_fit" ) : try : num_epochs = trainable . _impl_instance ( ) . num_epochs except AttributeError : if num_epochs_batching is None : warnings . warn ( "Operator {} does not have num_epochs and none given to Batching operator, using 1 as a default" . format ( trainable . name ( ) ) ) num_epochs = 1 else : num_epochs = num_epochs_batching assert num_epochs >= 0 elif trainable . is_frozen_trained ( ) : pass else : raise AttributeError ( "All operators to be trained with batching need to implement partial_fit. {} doesn't." . format ( operator . name ( ) ) ) inputs_for_transform : Any = inputs if trainable . is_frozen_trained ( ) : trained = trainable else : for epoch in range ( num_epochs ) : for _ , batch_data in enumerate ( inputs ) : # batching_transformer will output only one obj if isinstance ( batch_data , tuple ) : batch_X , batch_y = batch_data elif isinstance ( batch_data , list ) : batch_X = batch_data [ 0 ] batch_y = batch_data [ 1 ] else : batch_X = batch_data batch_y = None if trainable . is_supervised ( ) : try : trained = trainable . partial_fit ( batch_X , batch_y , classes = y ) except TypeError : trained = trainable . partial_fit ( batch_X , batch_y ) else : trained = trainable . partial_fit ( batch_X ) assert trained is not None trained = TrainedIndividualOp ( trained . name ( ) , trained . _impl , trained . _schemas , None , _lale_trained = True , ) trained_map [ operator ] = trained trained_steps . append ( trained ) output = None for batch_idx , batch_data in enumerate ( inputs_for_transform ) : # batching_transformer will output only one obj if isinstance ( batch_data , tuple ) : batch_X , batch_y = batch_data elif isinstance ( batch_data , list ) : batch_X = batch_data [ 0 ] batch_y = batch_data [ 1 ] else : batch_X = batch_data batch_y = None if trained . is_transformer ( ) : batch_output = trained . transform ( batch_X , batch_y ) else : if trainable in sink_nodes : batch_output = trained . _predict ( X = batch_X ) # We don't support y for predict yet as there is no compelling case else : # This is ok because trainable pipelines steps # must only be individual operators if trained . has_method ( "predict_proba" ) : # type: ignore batch_output = trained . predict_proba ( X = batch_X ) elif trained . has_method ( "decision_function" ) : # type: ignore batch_output = trained . decision_function ( X = batch_X ) else : batch_output = trained . _predict ( X = batch_X ) if isinstance ( batch_output , tuple ) : batch_out_X , batch_out_y = batch_output else : batch_out_X = batch_output batch_out_y = None if serialize : output = lale . helpers . write_batch_output_to_file ( output , os . path . join ( serialization_out_dir , "fit_with_batches" + str ( operator_idx ) + ".hdf5" , ) , len ( inputs_for_transform . dataset ) , batch_idx , batch_X , batch_y , batch_out_X , batch_out_y , ) else : if batch_out_y is None : output = lale . helpers . append_batch ( output , ( batch_output , batch_y ) ) else : output = lale . helpers . append_batch ( output , batch_output ) if serialize : output . close ( ) # type: ignore output = lale . helpers . create_data_loader ( os . path . join ( serialization_out_dir , "fit_with_batches" + str ( operator_idx ) + ".hdf5" , ) , batch_size = inputs_for_transform . batch_size , ) else : if isinstance ( output , tuple ) : output = lale . helpers . create_data_loader ( X = output [ 0 ] , y = output [ 1 ] , batch_size = inputs_for_transform . batch_size , ) else : output = lale . helpers . create_data_loader ( X = output , y = None , batch_size = inputs_for_transform . batch_size ) outputs [ operator ] = output operator_idx += 1 if serialize : shutil . rmtree ( serialization_out_dir ) trained_edges = [ ( trained_map [ x ] , trained_map [ y ] ) for ( x , y ) in edges ] trained_steps2 : Any = trained_steps result : TrainedPipeline [ TrainedIndividualOp ] = TrainedPipeline ( trained_steps2 , trained_edges , ordered = True , _lale_trained = True ) self . _trained = result return result