license import { TypedArray , util } from '@tensorflow/tfjs-core' ; function split ( str , delimiters , skipEmpty , result ) { if ( ! str . length ) { return ; } if ( delimiters . length === 0 ) { for ( let i = 0 ; i < str . length ; ++ i ) { result . push ( str . subarray ( i , i + 1 ) ) ; } return ; } if ( delimiters . length === 1 ) { const delimiter = delimiters [ 0 ] ; let f = str . indexOf ( delimiter ) ; while ( f !== - 1 ) { const token = str . subarray ( 0 , f ) ; if ( ! skipEmpty || token . length !== 0 ) { result . push ( token ) ; } str = str . subarray ( f + 1 ) ; f = str . indexOf ( delimiter ) ; } if ( ! skipEmpty || str . length !== 0 ) { result . push ( str ) ; } return ; } let tokenStart = 0 ; for ( let i = 0 ; i < str . length + 1 ; i ++ ) { if ( ( i === str . length ) || ( delimiters . indexOf ( str [ i ] ) !== - 1 ) ) { const token = str . subarray ( tokenStart , i ) ; if ( ! skipEmpty || token . length !== 0 ) { result . push ( token ) ; } tokenStart = i + 1 ; } } } export function stringSplitImpl ( input , delimiter , skipEmpty ) { const batchSize = input . length ; const tokens = [ ] ; let outputSize = 0 ; let maxNumEntries = 0 ; const numIndices = new Array ( batchSize ) ; for ( let i = 0 ; i < batchSize ; ++ i ) { const prevTokensLength = tokens . length ; split ( input [ i ] , delimiter , skipEmpty , tokens ) ; const nEntries = tokens . length - prevTokensLength ; numIndices [ i ] = nEntries ; outputSize += nEntries ; maxNumEntries = Math . max ( maxNumEntries , nEntries ) ; } const indices = util . getArrayFromDType ( 'int32' , outputSize * 2 ) as TypedArray ; const values = new Array ( outputSize ) ; const shape = [ batchSize , maxNumEntries ] ; let c = 0 ; for ( let i = 0 ; i < batchSize ; ++ i ) { for ( let j = 0 ; j < numIndices [ i ] ; ++ j ) { indices [ c * 2 ] = i ; indices [ c * 2 + 1 ] = j ; values [ c ] = tokens [ c ] ; ++ c ; } } return [ indices , values , shape ] ; }