% Generated by roxygen2: do not edit by hand
% Please edit documentation in R/PipeOpTextVectorizer.R
\name{mlr_pipeops_textvectorizer}
\alias{mlr_pipeops_textvectorizer}
\alias{PipeOpTextVectorizer}
\title{Bag-of-word Representation of Character Features}
\format{
\code{\link[R6:R6Class]{R6Class}} object inheriting from \code{\link{PipeOpTaskPreproc}}/\code{\link{PipeOp}}.
}
\description{
Computes a bag-of-word representation from a (set of) columns.
Columns of type \code{character} are split up into words.
Uses the \code{\link[quanteda:dfm]{quanteda::dfm()}},
\code{\link[quanteda:dfm_trim]{quanteda::dfm_trim()}} from the 'quanteda' package.
TF-IDF computation works similarly to \code{\link[quanteda:dfm_tfidf]{quanteda::dfm_tfidf()}}
but has been adjusted for train/test data split using \code{\link[quanteda:docfreq]{quanteda::docfreq()}}
and \code{\link[quanteda:dfm_weight]{quanteda::dfm_weight()}}

In short:
\itemize{
\item Per default, produces a bag-of-words representation
\item If \code{n} is set to values > 1, ngrams are computed
\item If \code{df_trim} parameters are set, the bag-of-words is trimmed.
\item The \code{scheme_tf} parameter controls term-frequency (per-document, i.e. per-row) weighting
\item The \code{scheme_df} parameter controls the document-frequency (per token, i.e. per-column) weighting.
}

Parameters specify arguments to quanteda's \code{dfm}, \code{dfm_trim}, \code{docfreq} and \code{dfm_weight}.
What belongs to what can be obtained from each params \code{tags} where \code{tokenizer} are
arguments passed on to \code{\link[quanteda:dfm]{quanteda::dfm()}}.
Defaults to a bag-of-words representation with token counts as matrix entries.

In order to perform the \emph{default} \code{dfm_tfidf} weighting, set the \code{scheme_df} parameter to \code{"inverse"}.
The \code{scheme_df} parameter is initialized to \code{"unary"}, which disables document frequency weighting.

The pipeop works as follows:
\enumerate{
\item Words are tokenized using \code{\link[quanteda:tokens]{quanteda::tokens}}.
\item Ngrams are computed using \code{\link[quanteda:tokens_ngrams]{quanteda::tokens_ngrams}}
\item A document-frequency matrix is computed using \code{\link[quanteda:dfm]{quanteda::dfm}}
\item The document-frequency matrix is trimmed using \code{\link[quanteda:dfm_trim]{quanteda::dfm_trim}} during train-time.
\item The document-frequency matrix is re-weighted (similar to \code{\link[quanteda:dfm_tfidf]{quanteda::dfm_tfidf}}) if \code{scheme_df} is not set to \code{"unary"}.
}
}
\section{Construction}{


\if{html}{\out{<div class="sourceCode">}}\preformatted{PipeOpTextVectorizer$new(id = "textvectorizer", param_vals = list())
}\if{html}{\out{</div>}}
\itemize{
\item \code{id} :: \code{character(1)}\cr
Identifier of resulting object, default \code{"textvectorizer"}.
\item \code{param_vals} :: named \code{list}\cr
List of hyperparameter settings, overwriting the hyperparameter settings that would otherwise be set during construction. Default \code{list()}.
}
}

\section{Input and Output Channels}{

Input and output channels are inherited from \code{\link{PipeOpTaskPreproc}}.

The output is the input \code{\link[mlr3:Task]{Task}} with all affected features converted to a bag-of-words
representation.
}

\section{State}{

The \verb{$state} is a list with element 'cols': A vector of extracted columns.
}

\section{Parameters}{

The parameters are the parameters inherited from \code{\link{PipeOpTaskPreproc}}, as well as:
\itemize{
\item \code{return_type} :: \code{character(1)}\cr
Whether to return an integer representation ("integer-sequence") or a Bag-of-words ("bow").
If set to "integer_sequence", tokens are replaced by an integer and padded/truncated to \code{sequence_length}.
If set to "factor_sequence", tokens are replaced by a factor and padded/truncated to \code{sequence_length}.
If set to 'bow', a possibly weighted bag-of-words matrix is returned.
Defaults to \code{bow}.
\item \code{stopwords_language} :: \code{character(1)}\cr
Language to use for stopword filtering. Needs to be either \code{"none"}, a language identifier listed in
\code{stopwords::stopwords_getlanguages("snowball")} (\code{"de"}, \code{"en"}, ...) or \code{"smart"}.
\code{"none"} disables language-specific stopwords.
\code{"smart"} coresponds to \code{stopwords::stopwords(source = "smart")}, which
contains \emph{English} stopwords and also removes one-character strings. Initialized to \code{"smart"}.\cr
\item \code{extra_stopwords} :: \code{character}\cr
Extra stopwords to remove. Must be a \code{character} vector containing individual tokens to remove. Initialized to \code{character(0)}.
When \code{n} is set to values greater than 1, this can also contain stop-ngrams.
\item \code{tolower} :: \code{logical(1)}\cr
Convert to lower case? See \code{\link[quanteda:dfm]{quanteda::dfm}}. Default: \code{TRUE}.
\item \code{stem} :: \code{logical(1)}\cr
Perform stemming? See \code{\link[quanteda:dfm]{quanteda::dfm}}. Default: \code{FALSE}.
\item \code{what} :: \code{character(1)}\cr
Tokenization splitter. See \code{\link[quanteda:tokens]{quanteda::tokens}}. Default: \code{word}.
\item \code{remove_punct} :: \code{logical(1)}\cr
See \code{\link[quanteda:tokens]{quanteda::tokens}}. Default: \code{FALSE}.
\item \code{remove_url} :: \code{logical(1)}\cr
See \code{\link[quanteda:tokens]{quanteda::tokens}}. Default: \code{FALSE}.
\item \code{remove_symbols} :: \code{logical(1)}\cr
See \code{\link[quanteda:tokens]{quanteda::tokens}}. Default: \code{FALSE}.
\item \code{remove_numbers} :: \code{logical(1)}\cr
See \code{\link[quanteda:tokens]{quanteda::tokens}}. Default: \code{FALSE}.
\item \code{remove_separators} :: \code{logical(1)}\cr
See \code{\link[quanteda:tokens]{quanteda::tokens}}. Default: \code{TRUE}.
\item \code{split_hypens} :: \code{logical(1)}\cr
See \code{\link[quanteda:tokens]{quanteda::tokens}}. Default: \code{FALSE}.
\item \code{n} :: \code{integer}\cr
Vector of ngram lengths. See \code{\link[quanteda:tokens_ngrams]{quanteda::tokens_ngrams}}. Initialized to 1, deviating from the base function's default.
Note that this can be a \emph{vector} of multiple values, to construct ngrams of multiple orders.
\item \code{skip} :: \code{integer}\cr
Vector of skips. See \code{\link[quanteda:tokens_ngrams]{quanteda::tokens_ngrams}}. Default: 0. Note that this can be a \emph{vector} of multiple values.
\item \code{sparsity} :: \code{numeric(1)}\cr
Desired sparsity of the 'tfm' matrix. See \code{\link[quanteda:dfm_trim]{quanteda::dfm_trim}}. Default: \code{NULL}.
\item \code{max_termfreq} :: \code{numeric(1)}\cr
Maximum term frequency in the 'tfm' matrix. See \code{\link[quanteda:dfm_trim]{quanteda::dfm_trim}}. Default: \code{NULL}.
\item \code{min_termfreq} :: \code{numeric(1)}\cr
Minimum term frequency in the 'tfm' matrix. See \code{\link[quanteda:dfm_trim]{quanteda::dfm_trim}}. Default: \code{NULL}.
\item \code{termfreq_type} :: \code{character(1)}\cr
How to asess term frequency. See \code{\link[quanteda:dfm_trim]{quanteda::dfm_trim}}. Default: \code{"count"}.
\item \code{scheme_df} :: \code{character(1)} \cr
Weighting scheme for document frequency: See \code{\link[quanteda:docfreq]{quanteda::docfreq}}. Initialized to \code{"unary"} (1 for each document, deviating from base function default).
\item \code{smoothing_df} :: \code{numeric(1)}\cr
See \code{\link[quanteda:docfreq]{quanteda::docfreq}}. Default: 0.
\item \code{k_df} :: \code{numeric(1)}\cr
\code{k} parameter given to \code{\link[quanteda:docfreq]{quanteda::docfreq}} (see there).
Default is 0.
\item \code{threshold_df} :: \code{numeric(1)}\cr
See \code{\link[quanteda:docfreq]{quanteda::docfreq}}. Default: 0. Only considered for \code{scheme_df} = \code{"count"}.
\item \code{base_df} :: \code{numeric(1)}\cr
The base for logarithms in \code{\link[quanteda:docfreq]{quanteda::docfreq}} (see there). Default: 10.
\item \code{scheme_tf} :: \code{character(1)} \cr
Weighting scheme for term frequency: See \code{\link[quanteda:dfm_weight]{quanteda::dfm_weight}}. Default: \code{"count"}.
\item \code{k_tf} :: \code{numeric(1)}\cr
\code{k} parameter given to \code{\link[quanteda:dfm_weight]{quanteda::dfm_weight}} (see there).
Default behaviour is 0.5.
\item \code{base_df} :: \code{numeric(1)}\cr
The base for logarithms in \code{\link[quanteda:dfm_weight]{quanteda::dfm_weight}} (see there). Default: 10.
}

#' * \code{sequence_length} :: \code{integer(1)}\cr
The length of the integer sequence. Defaults to \code{Inf}, i.e. all texts are padded to the length
of the longest text. Only relevant for "return_type" : "integer_sequence"
}

\section{Internals}{

See Description. Internally uses the \code{quanteda} package. Calls \code{\link[quanteda:tokens]{quanteda::tokens}}, \code{\link[quanteda:tokens_ngrams]{quanteda::tokens_ngrams}} and \code{\link[quanteda:dfm]{quanteda::dfm}}. During training,
\code{\link[quanteda:dfm_trim]{quanteda::dfm_trim}} is also called. Tokens not seen during training are dropped during prediction.
}

\section{Methods}{

Only methods inherited from \code{\link{PipeOpTaskPreproc}}/\code{\link{PipeOp}}.
}

\examples{
\dontshow{if (mlr3misc::require_namespaces(c("stopwords", "quanteda"), quietly = TRUE)) (if (getRversion() >= "3.4") withAutoprint else force)(\{ # examplesIf}
library("mlr3")
library("data.table")
# create some text data
dt = data.table(
  txt = replicate(150, paste0(sample(letters, 3), collapse = " "))
)
task = tsk("iris")$cbind(dt)

pos = po("textvectorizer", param_vals = list(stopwords_language = "en"))

pos$train(list(task))[[1]]$data()

one_line_of_iris = task$filter(13)

one_line_of_iris$data()

pos$predict(list(one_line_of_iris))[[1]]$data()
\dontshow{\}) # examplesIf}
}
\seealso{
https://mlr-org.com/pipeops.html

Other PipeOps: 
\code{\link{PipeOp}},
\code{\link{PipeOpEnsemble}},
\code{\link{PipeOpImpute}},
\code{\link{PipeOpTargetTrafo}},
\code{\link{PipeOpTaskPreproc}},
\code{\link{PipeOpTaskPreprocSimple}},
\code{\link{mlr_pipeops}},
\code{\link{mlr_pipeops_adas}},
\code{\link{mlr_pipeops_blsmote}},
\code{\link{mlr_pipeops_boxcox}},
\code{\link{mlr_pipeops_branch}},
\code{\link{mlr_pipeops_chunk}},
\code{\link{mlr_pipeops_classbalancing}},
\code{\link{mlr_pipeops_classifavg}},
\code{\link{mlr_pipeops_classweights}},
\code{\link{mlr_pipeops_colapply}},
\code{\link{mlr_pipeops_collapsefactors}},
\code{\link{mlr_pipeops_colroles}},
\code{\link{mlr_pipeops_copy}},
\code{\link{mlr_pipeops_datefeatures}},
\code{\link{mlr_pipeops_encode}},
\code{\link{mlr_pipeops_encodeimpact}},
\code{\link{mlr_pipeops_encodelmer}},
\code{\link{mlr_pipeops_featureunion}},
\code{\link{mlr_pipeops_filter}},
\code{\link{mlr_pipeops_fixfactors}},
\code{\link{mlr_pipeops_histbin}},
\code{\link{mlr_pipeops_ica}},
\code{\link{mlr_pipeops_imputeconstant}},
\code{\link{mlr_pipeops_imputehist}},
\code{\link{mlr_pipeops_imputelearner}},
\code{\link{mlr_pipeops_imputemean}},
\code{\link{mlr_pipeops_imputemedian}},
\code{\link{mlr_pipeops_imputemode}},
\code{\link{mlr_pipeops_imputeoor}},
\code{\link{mlr_pipeops_imputesample}},
\code{\link{mlr_pipeops_kernelpca}},
\code{\link{mlr_pipeops_learner}},
\code{\link{mlr_pipeops_missind}},
\code{\link{mlr_pipeops_modelmatrix}},
\code{\link{mlr_pipeops_multiplicityexply}},
\code{\link{mlr_pipeops_multiplicityimply}},
\code{\link{mlr_pipeops_mutate}},
\code{\link{mlr_pipeops_nearmiss}},
\code{\link{mlr_pipeops_nmf}},
\code{\link{mlr_pipeops_nop}},
\code{\link{mlr_pipeops_ovrsplit}},
\code{\link{mlr_pipeops_ovrunite}},
\code{\link{mlr_pipeops_pca}},
\code{\link{mlr_pipeops_proxy}},
\code{\link{mlr_pipeops_quantilebin}},
\code{\link{mlr_pipeops_randomprojection}},
\code{\link{mlr_pipeops_randomresponse}},
\code{\link{mlr_pipeops_regravg}},
\code{\link{mlr_pipeops_removeconstants}},
\code{\link{mlr_pipeops_renamecolumns}},
\code{\link{mlr_pipeops_replicate}},
\code{\link{mlr_pipeops_rowapply}},
\code{\link{mlr_pipeops_scale}},
\code{\link{mlr_pipeops_scalemaxabs}},
\code{\link{mlr_pipeops_scalerange}},
\code{\link{mlr_pipeops_select}},
\code{\link{mlr_pipeops_smote}},
\code{\link{mlr_pipeops_smotenc}},
\code{\link{mlr_pipeops_spatialsign}},
\code{\link{mlr_pipeops_subsample}},
\code{\link{mlr_pipeops_targetinvert}},
\code{\link{mlr_pipeops_targetmutate}},
\code{\link{mlr_pipeops_targettrafoscalerange}},
\code{\link{mlr_pipeops_threshold}},
\code{\link{mlr_pipeops_tomek}},
\code{\link{mlr_pipeops_tunethreshold}},
\code{\link{mlr_pipeops_unbranch}},
\code{\link{mlr_pipeops_updatetarget}},
\code{\link{mlr_pipeops_vtreat}},
\code{\link{mlr_pipeops_yeojohnson}}
}
\concept{PipeOps}
