license import * as tf from '@tensorflow/tfjs-core' ; import * as tfl from '@tensorflow/tfjs-layers' ; /**
 * This file defines the interfaces related to SpeechCommandRecognizer.
 */ export type FFT_TYPE = 'BROWSER_FFT' | 'SOFT_FFT' ; export type RecognizerCallback = ( result : SpeechCommandRecognizerResult ) => Promise < void > ; /**
 * Interface for a speech-command recognizer.
 */ export interface SpeechCommandRecognizer { /**
     * Load the underlying model instance and associated metadata.
     *
     * If the model and the metadata are already loaded, do nothing.
     */ ensureModelLoaded ( ) : Promise < void > ; param callback param config throws listen ( callback , config ? ) : Promise < void > ; throws stopListening ( ) : Promise < void > ; /**
     * Check if this instance is currently performing
     * streaming recognition.
     */ isListening ( ) : boolean ; param input returns throws recognize ( input ? , config ? ) : Promise < SpeechCommandRecognizerResult > ; /**
     * Get the input shape of the tf.Model the underlies the recognizer.
     */ modelInputShape ( ) : tfl . Shape ; /**
     * Getter for word labels.
     *
     * The word labels are an alphabetically sorted Array of strings.
     */ wordLabels ( ) : string [ ] ; /**
     * Get the parameters such as the required number of frames.
     */ params ( ) : RecognizerParams ; param name returns createTransfer ( name ) : TransferSpeechCommandRecognizer ; } export interface ExampleCollectionOptions { /**
     * Multiplier for the duration.
     *
     * This is the ratio between the duration of the to-be-collected
     * example and the duration of each input example accepted by the
     * underlying convnet.
     *
     * If not provided, will default to 1.
     *
     * Must be a number >=1.
     */ durationMultiplier ? : number ; /**
     * Duration in seconds.
     *
     * Mutually exclusive with durationMultiplier.
     * If specified, must be >0.
     */ durationSec ? : number ; /**
     * Optional constraints for the audio track.
     *
     * E.g., this can be used to select a microphone when multiple microphones
     * are available on the system: `{deviceId: 'deadbeef'}`.
     */ audioTrackConstraints ? : MediaTrackConstraints ; /**
     * Optional snipppet duration in seconds.
     *
     * Must be supplied if `onSnippet` is specified.
     */ snippetDurationSec ? : number ; /**
     * Optional snippet callback.
     *
     * Must be provided if `snippetDurationSec` is specified.
     *
     * Gets called every snippetDurationSec with a latest slice of the
     * spectrogram. It is the spectrogram accumulated since the last invocation of
     * the callback (or for the first time, since when `collectExample()` is
     * started).
     */ onSnippet ? : ( spectrogram : SpectrogramData ) => Promise < void > ; /**
     * Whether to collect the raw time-domain audio waveform in addition to the
     * spectrogram.
     *
     * Default: `false`.
     */ includeRawAudio ? : boolean ; } /**
 * Metadata for a speech-comamnds recognizer.
 */ export interface SpeechCommandRecognizerMetadata { /** Version of the speech-commands library. */ tfjsSpeechCommandsVersion : string ; /** Name of the model. */ modelName ? : string ; /** A time stamp for when this metadata is generatd. */ timeStamp ? : string ; /**
     * Word labels for the recognizer model's output probability scores.
     *
     * The length of this array should be equal to the size of the last dimension
     * of the model's output.
     */ wordLabels : string [ ] ; } /**
 * Interface for a transfer-learning speech command recognizer.
 *
 * This inherits the `SpeechCommandRecognizer`. It adds methods for
 * collecting and clearing examples for transfer learning, methods for
 * querying the status of example collection, and for performing the
 * transfer-learning training.
 */ export interface TransferSpeechCommandRecognizer extends SpeechCommandRecognizer { param { string } word returns { SpectrogramData } throws collectExample ( word , options ? ) : Promise < SpectrogramData > ; /**
     * Clear all transfer learning examples collected so far.
     */ clearExamples ( ) : void ; returns { { [ word : string ] : number } } countExamples ( ) : { [ word ] : number ; } ; param config { TransferLearnConfig } returns { tf . History } throws train ( config ? ) : Promise < tfl . History | [ tfl . History , tfl . History ] > ; param config returns evaluate ( config ) : Promise < EvaluateResult > ; param label returns getExamples ( label ) : Array < { uid : string ; example : Example ; } > ; /** Set the key frame index of a given example. */ setExampleKeyFrameIndex ( uid , keyFrameIndex ) : void ; param serialized param clearExisting loadExamples ( serialized , clearExisting ? ) : void ; param wordLabels returns serializeExamples ( wordLabels ? ) : ArrayBuffer ; param uid removeExample ( uid ) : void ; returns isDatasetEmpty ( ) : boolean ; param handlerOrURL returns save ( handlerOrURL ? ) : Promise < tf . io . SaveResult > ; param handlerOrURL load ( handlerOrURL ? ) : Promise < void > ; /**
     * Get metadata about the transfer recognizer.
     *
     * The metadata includes but is not limited to: speech-commands library
     * version, word labels that correspond to the model's probability outputs.
     */ getMetadata ( ) : SpeechCommandRecognizerMetadata ; } /**
 * Interface for a snippet of audio spectrogram.
 */ export interface SpectrogramData { /**
     * The float32 data for the spectrogram.
     *
     * Stored frame by frame. For example, the first N elements
     * belong to the first time frame and the next N elements belong
     * to the second time frame, and so forth.
     */ data : Float32Array ; /**
     * Number of points per frame, i.e., FFT length per frame.
     */ frameSize : number ; /**
     * Duration of each frame in milliseconds.
     */ frameDurationMillis ? : number ; /**
     * Index to the key frame (0-based).
     *
     * A key frame is a frame in the spectrogram that belongs to
     * the utterance of interest. It is used to distinguish the
     * utterance part from the background-noise part.
     *
     * A typical use of key frame index: when multiple training examples are
     * extracted from a spectroram, every example is guaranteed to include
     * the key frame.
     *
     * Key frame is not required. If it is missing, heuristics algorithms
     * (e.g., finding the highest-intensity frame) can be used to calculate
     * the key frame.
     */ keyFrameIndex ? : number ; } /**
 * Interface for a result emitted by a speech-command recognizer.
 *
 * It is used in the callback of a recognizer's streaming or offline
 * recognition method. It represents the result for a short snippet of
 * audio.
 */ export interface SpeechCommandRecognizerResult { /**
     * Probability scores for the words.
     */ scores : Float32Array | Float32Array [ ] ; /**
     * Optional spectrogram data.
     */ spectrogram ? : SpectrogramData ; /**
     * Embedding (internal activation) for the input.
     *
     * This field is populated if and only if `includeEmbedding`
     * is `true` in the configuration object used during the `recognize` call.
     */ embedding ? : tf . Tensor ; } export interface StreamingRecognitionConfig { /**
     * Overlap factor. Must be >=0 and <1.
     * Defaults to 0.5.
     * For example, if the model takes a frame length of 1000 ms,
     * and if overlap factor is 0.4, there will be a 400ms
     * overlap between two successive frames, i.e., frames
     * will be taken every 600 ms.
     */ overlapFactor ? : number ; /**
     * Amount to time in ms to suppress recognizer after a word is recognized.
     *
     * Defaults to 1000 ms.
     */ suppressionTimeMillis ? : number ; /**
     * Threshold for the maximum probability value in a model prediction
     * output to be greater than or equal to, below which the callback
     * will not be called.
     *
     * Must be a number >=0 and <=1.
     *
     * The value will be overridden to `0` if `includeEmbedding` is `true`.
     *
     * If `null` or `undefined`, will default to `0`.
     */ probabilityThreshold ? : number ; /**
     * Invoke the callback for background noise and unknown.
     *
     * The value will be overridden to `true` if `includeEmbedding` is `true`.
     *
     * Default: `false`.
     */ invokeCallbackOnNoiseAndUnknown ? : boolean ; /**
     * Whether the spectrogram is to be provided in the each recognition
     * callback call.
     *
     * Default: `false`.
     */ includeSpectrogram ? : boolean ; /**
     * Whether to include the embedding (internal activation).
     *
     * If set as `true`, the values of the following configuration fields
     * in this object will be overridden:
     *
     * - `probabilityThreshold` will be overridden to 0.
     * - `invokeCallbackOnNoiseAndUnknown` will be overridden to `true`.
     *
     * Default: `false`.
     */ includeEmbedding ? : boolean ; /**
     * Optional constraints for the audio track.
     *
     * E.g., this can be used to select a microphone when multiple microphones
     * are available on the system: `{deviceId: 'deadbeef'}`.
     */ audioTrackConstraints ? : MediaTrackConstraints ; } export interface RecognizeConfig { /**
     * Whether the spectrogram is to be provided in the each recognition
     * callback call.
     *
     * Default: `false`.
     */ includeSpectrogram ? : boolean ; /**
     * Whether to include the embedding (internal activation).
     *
     * Default: `false`.
     */ includeEmbedding ? : boolean ; } export interface AudioDataAugmentationOptions { /**
     * Additive ratio for augmenting the data by mixing the word spectrograms
     * with background-noise ones.
     *
     * If not `null` or `undefined`, will cause extra word spectrograms to be
     * created through the equation:
     *   (normalizedWordSpectrogram +
     *    augmentByMixingNoiseRatio * normalizedNoiseSpectrogram)
     *
     * The normalizedNoiseSpectrogram will be drawn randomly from all noise
     * snippets available. If no noise snippet is available, an Error will
     * be thrown.
     *
     * Default: `undefined`.
     */ augmentByMixingNoiseRatio ? : number ; } /**
 * Configurations for the training of a transfer-learning recognizer.
 *
 * It is used during calls to the `TransferSpeechCommandRecognizer.train()`
 * method.
 */ export interface TransferLearnConfig extends AudioDataAugmentationOptions { /**
     * Number of training epochs (default: 20).
     */ epochs ? : number ; /**
     * Optimizer to be used for training (default: 'sgd').
     */ optimizer ? : string | tf . Optimizer ; /**
     * Batch size of training (default: 128).
     */ batchSize ? : number ; /**
     * Validation split to be used during training.
     *
     * Default: null (no validation split).
     *
     * Note that this is split is different from the basic validation-split
     * paradigm in TensorFlow.js. It makes sure that the distribution of the
     * classes in the training and validation sets are approximately balanced.
     *
     * If specified, must be a number > 0 and < 1.
     */ validationSplit ? : number ; /**
     * Number of fine-tuning epochs to run after the initial `epochs` epochs
     * of transfer-learning training.
     *
     * During the fine-tuning, the last dense layer of the truncated base
     * model (i.e., the second-last dense layer of the original model) is
     * unfrozen and updated through backpropagation.
     *
     * If specified, must be an integer > 0.
     */ fineTuningEpochs ? : number ; /**
     * The optimizer for fine-tuning after the initial transfer-learning
     * training.
     *
     * This parameter is used only if `fineTuningEpochs` is specified
     * and is a positive integre.
     *
     * Default: 'sgd'.
     */ fineTuningOptimizer ? : string | tf . Optimizer ; /**
     * tf.Callback to be used during the initial training (i.e., not
     * the fine-tuning phase).
     */ callback ? : tfl . CustomCallbackArgs ; /**
     * tf.Callback to be used durnig the fine-tuning phase.
     *
     * This parameter is used only if `fineTuningEpochs` is specified
     * and is a positive integer.
     */ fineTuningCallback ? : tfl . CustomCallbackArgs ; /**
     * Ratio between the window hop and the window width.
     *
     * Used during extraction of multiple spectrograms matching the underlying
     * model's input shape from a longer spectroram.
     *
     * Defaults to 0.25.
     *
     * For example, if the spectrogram window accepted by the underlying model
     * is 43 frames long, then the default windowHopRatio 0.25 will lead to
     * a hop of Math.round(43 * 0.25) = 11 frames.
     */ windowHopRatio ? : number ; /**
     * The threshold for the total duration of the dataset above which
     * `fitDataset()` will be used in lieu of `fit()`.
     *
     * Default: 60e3 (1 minute).
     */ fitDatasetDurationMillisThreshold ? : number ; } /**
 * Type for a Receiver Operating Characteristics (ROC) curve.
 */ export type ROCCurve = Array < { probThreshold ? : number ; fpr : number ; tpr : number ; falsePositivesPerHour ? : number ; } > ; /**
 * Model evaluation result.
 */ export interface EvaluateResult { /**
     * ROC curve.
     */ rocCurve ? : ROCCurve ; /**
     * Area under the (ROC) curve.
     */ auc ? : number ; } /**
 * Model evaluation configuration.
 */ export interface EvaluateConfig { /**
     * Ratio between the window hop and the window width.
     *
     * Used during extraction of multiple spectrograms matching the underlying
     * model's input shape from a longer spectroram.
     *
     * For example, if the spectrogram window accepted by the underlying model
     * is 43 frames long, then the default windowHopRatio 0.25 will lead to
     * a hop of Math.round(43 * 0.25) = 11 frames.
     */ windowHopRatio : number ; /**
     * Word probability score thresholds, used to calculate the ROC.
     *
     * E.g., [0, 0.2, 0.4, 0.6, 0.8, 1.0].
     */ wordProbThresholds : number [ ] ; } /**
 * Parameters for a speech-command recognizer.
 */ export interface RecognizerParams { /**
     * Total duration per spectragram, in milliseconds.
     */ spectrogramDurationMillis ? : number ; /**
     * FFT encoding size per spectrogram column.
     */ fftSize ? : number ; /**
     * Sampling rate, in Hz.
     */ sampleRateHz ? : number ; } /**
 * Interface of an audio feature extractor.
 */ export interface FeatureExtractor { /**
     * Config the feature extractor.
     */ setConfig ( params ) : void ; /**
     * Start the feature extraction from the audio samples.
     */ start ( audioTrackConstraints ? ) : Promise < Float32Array [ ] | void > ; /**
     * Stop the feature extraction.
     */ stop ( ) : Promise < void > ; /**
     * Get the extractor features collected since last call.
     */ getFeatures ( ) : Float32Array [ ] ; } /** Snippet of pulse-code modulation (PCM) audio data. */ export interface RawAudioData { /** Samples of the snippet. */ data : Float32Array ; /** Sampling rate, in Hz. */ sampleRateHz : number ; } /**
 * A short, labeled snippet of speech or audio.
 *
 * This can be used for training a transfer model based on the base
 * speech-commands model, among other things.
 *
 * A set of `Example`s can make up a dataset.
 */ export interface Example { /** A label for the example. */ label : string ; /** Spectrogram data. */ spectrogram : SpectrogramData ; /**
     * Raw audio in PCM (pulse-code modulation) format.
     *
     * Optional.
     */ rawAudio ? : RawAudioData ; }