(self,  # type: ignore
                         tokens: List[str],
                         token_representations: FloatTensor = None,
                         labels: List[str] = None)
return Instance(fields)
We take `pre-tokenized` input here, because we don't have a tokenizer in this class.

        Parameters
        ----------
        tokens : ``List[str]``, required.
            The tokens in a given sentence.
        token_representations: ``FloatTensor``, optional (default=``None``)
            Precomputed token representations to use in the instance. If ``None``,
            we use a ``Contextualizer`` provided to the dataset reader to calculate
            the token representations. Shape is (seq_len, representation_dim).
        labels : ``List[str]``, optional, (default = None).
            The labels for the words in the sentence.

        Returns
        -------
        An ``Instance`` containing the following fields:
            raw_tokens : ListField[MetadataField]
                The raw string tokens in the sentence. Each MetadataField stores the raw string
                of a single token.
            token_representations : ``ArrayField``
                Stores the representations for each token. Shape is (seq_len, represenatation_dim)
            labels : ``SequenceLabelField``
                The labels (only if supplied)
We take `pre-tokenized` input here, because we don't have a tokenizer in this class.
We take pre - tokenized input here because we don t have a tokenizer in this class .
def text_to_instance(self,  # type: ignore
                         tokens: List[str],
                         token_representations: FloatTensor = None,
                         labels: List[str] = None) -> Instance:
        """
        We take `pre-tokenized` input here, because we don't have a tokenizer in this class.

        Parameters
        ----------
        tokens : ``List[str]``, required.
            The tokens in a given sentence.
        token_representations: ``FloatTensor``, optional (default=``None``)
            Precomputed token representations to use in the instance. If ``None``,
            we use a ``Contextualizer`` provided to the dataset reader to calculate
            the token representations. Shape is (seq_len, representation_dim).
        labels : ``List[str]``, optional, (default = None).
            The labels for the words in the sentence.

        Returns
        -------
        An ``Instance`` containing the following fields:
            raw_tokens : ListField[MetadataField]
                The raw string tokens in the sentence. Each MetadataField stores the raw string
                of a single token.
            token_representations : ``ArrayField``
                Stores the representations for each token. Shape is (seq_len, represenatation_dim)
            labels : ``SequenceLabelField``
                The labels (only if supplied)
        """
        # pylint: disable=arguments-differ
        raw_text_field = ListField([MetadataField(token) for token in tokens])
        fields: Dict[str, Field] = {"raw_tokens": raw_text_field}

        if token_representations is None and self._contextualizer:
            # Contextualize the tokens
            token_representations = self._contextualizer(tokens)
        # Add representations for the tokens in the sequence to the field.
        if token_representations is not None:
            fields["token_representations"] = ArrayField(
                token_representations.numpy())

        if labels is not None:
            fields["labels"] = SequenceLabelField(labels, raw_text_field)
        return Instance(fields)
def text_to_instance ( self , # type: ignore tokens : List [ str ] , token_representations : FloatTensor = None , labels : List [ str ] = None ) -> Instance : # pylint: disable=arguments-differ raw_text_field = ListField ( [ MetadataField ( token ) for token in tokens ] ) fields : Dict [ str , Field ] = { "raw_tokens" : raw_text_field } if token_representations is None and self . _contextualizer : # Contextualize the tokens token_representations = self . _contextualizer ( tokens ) # Add representations for the tokens in the sequence to the field. if token_representations is not None : fields [ "token_representations" ] = ArrayField ( token_representations . numpy ( ) ) if labels is not None : fields [ "labels" ] = SequenceLabelField ( labels , raw_text_field ) return Instance ( fields )