license import { Tensor , test_util } from '@tensorflow/tfjs-core' ; import * as tfOps from '@tensorflow/tfjs-core/dist/ops/ops_for_converter' ; import { ExecutionContext } from '../../executor/execution_context' ; import * as string from '../op_list/string' ; import { Node } from '../types' ; import { executeOp } from './string_executor' ; import { createBoolAttr , createNumberAttr , createNumericArrayAttr , createStrAttr , createTensorAttr , validateParam } from './test_helper' ; describe ( 'string' , ( ) => { let node ; const context = new ExecutionContext ( { } , { } , { } ) ; beforeEach ( ( ) => { node = { name : 'test' , op : '' , category : 'string' , inputNames : [ ] , inputs : [ ] , inputParams : { } , attrParams : { } , children : [ ] , outputs : [ ] } ; } ) ; describe ( 'executeOp' , ( ) => { describe ( 'StringNGrams' , ( ) => { it ( 'should call tfOps.string.stringNGrams' , async ( ) => { spyOn ( tfOps . string , 'stringNGrams' ) . and . callThrough ( ) ; node . op = 'StringNGrams' ; node . inputParams = { data : createTensorAttr ( 0 ) , dataSplits : createTensorAttr ( 1 ) } ; node . attrParams = { separator : createStrAttr ( '|' ) , nGramWidths : createNumericArrayAttr ( [ 3 ] ) , leftPad : createStrAttr ( 'LP' ) , rightPad : createStrAttr ( 'RP' ) , padWidth : createNumberAttr ( - 1 ) , preserveShortSequences : createBoolAttr ( false ) } ; node . inputNames = [ 'data' , 'dataSplits' ] ; node . outputs = [ 'ngrams' , 'ngrams_splits' ] ; const data = [ tfOps . tensor1d ( [ 'a' , 'b' , 'c' , 'd' , 'e' , 'f' ] , 'string' ) ] ; const dataSplits = [ tfOps . tensor1d ( [ 0 , 4 , 6 ] , 'int32' ) ] ; const result = executeOp ( node , { data , dataSplits } , context ) as Tensor [ ] ; expect ( tfOps . string . stringNGrams ) . toHaveBeenCalledWith ( data [ 0 ] , dataSplits [ 0 ] , '|' , [ 3 ] , 'LP' , 'RP' , - 1 , false ) ; test_util . expectArraysEqual ( await result [ 0 ] . data ( ) , [ 'LP|LP|a' , 'LP|a|b' , 'a|b|c' , 'b|c|d' , 'c|d|RP' , 'd|RP|RP' , 'LP|LP|e' , 'LP|e|f' , 'e|f|RP' , 'f|RP|RP' ] ) ; test_util . expectArraysEqual ( await result [ 1 ] . data ( ) , [ 0 , 6 , 10 ] ) ; } ) ; it ( 'should match json def' , ( ) => { node . op = 'StringNGrams' ; node . inputParams = { data : createTensorAttr ( 0 ) , dataSplits : createTensorAttr ( 1 ) } ; node . outputs = [ 'ngrams' , 'ngrams_splits' ] ; expect ( validateParam ( node , string . json ) ) . toBeTruthy ( ) ; } ) ; } ) ; describe ( 'StringSplit' , ( ) => { it ( 'should call tfOps.string.stringSplit' , async ( ) => { spyOn ( tfOps . string , 'stringSplit' ) . and . callThrough ( ) ; node . op = 'StringSplit' ; node . inputParams = { input : createTensorAttr ( 0 ) , delimiter : createTensorAttr ( 1 ) } ; node . attrParams = { skipEmpty : createBoolAttr ( false ) } ; node . inputNames = [ 'input' , 'delimiter' ] ; node . outputs = [ 'indices' , 'values' , 'shape' ] ; const input = [ tfOps . tensor1d ( [ '#a' , 'b#' , '#c#' ] , 'string' ) ] ; const delimiter = [ tfOps . scalar ( '#' , 'string' ) ] ; const result = executeOp ( node , { input , delimiter } , context ) as Tensor [ ] ; expect ( tfOps . string . stringSplit ) . toHaveBeenCalledWith ( input [ 0 ] , delimiter [ 0 ] , false ) ; test_util . expectArraysEqual ( await result [ 0 ] . data ( ) , [ 0 , 0 , 0 , 1 , 1 , 0 , 1 , 1 , 2 , 0 , 2 , 1 , 2 , 2 ] ) ; test_util . expectArraysEqual ( await result [ 1 ] . data ( ) , [ '' , 'a' , 'b' , '' , '' , 'c' , '' ] ) ; test_util . expectArraysEqual ( await result [ 2 ] . data ( ) , [ 3 , 3 ] ) ; } ) ; it ( 'should match json def' , ( ) => { node . op = 'StringSplit' ; node . inputParams = { input : createTensorAttr ( 0 ) , delimiter : createTensorAttr ( 1 ) } ; node . outputs = [ 'indices' , 'values' , 'shape' ] ; expect ( validateParam ( node , string . json ) ) . toBeTruthy ( ) ; } ) ; } ) ; describe ( 'StringToHashBucketFast' , ( ) => { it ( 'should call tfOps.string.stringToHashBucketFast' , async ( ) => { spyOn ( tfOps . string , 'stringToHashBucketFast' ) . and . callThrough ( ) ; node . op = 'StringToHashBucketFast' ; node . inputParams = { input : createTensorAttr ( 0 ) } ; node . attrParams = { numBuckets : createNumberAttr ( 10 ) } ; node . inputNames = [ 'input' ] ; const input = [ tfOps . tensor1d ( [ 'a' , 'b' , 'c' , 'd' ] , 'string' ) ] ; const result = executeOp ( node , { input } , context ) as Tensor [ ] ; expect ( tfOps . string . stringToHashBucketFast ) . toHaveBeenCalledWith ( input [ 0 ] , 10 ) ; test_util . expectArraysClose ( await result [ 0 ] . data ( ) , [ 9 , 2 , 2 , 5 ] ) ; } ) ; it ( 'should match json def' , ( ) => { node . op = 'StringToHashBucketFast' ; node . inputParams = { input : createTensorAttr ( 0 ) } ; expect ( validateParam ( node , string . json ) ) . toBeTruthy ( ) ; } ) ; } ) ; } ) ; } ) ;