import dlBin_ from 'datalib/src/bins/bins' ; import { inferAll } from 'datalib/src/import/type' ; import { summary } from 'datalib/src/stats' ; import { autoMaxBins } from 'vega-lite/build/src/bin' ; import { Channel } from 'vega-lite/build/src/channel' ; import { SingleTimeUnit , LocalSingleTimeUnit , isUTCTimeUnit , containsTimeUnit , TimeUnit , TIMEUNIT_PARTS , } from 'vega-lite/build/src/timeunit' ; import * as TYPE from 'vega-lite/build/src/type' ; import * as vegaTime from 'vega-time' ; import { DEFAULT_QUERY_CONFIG , QueryConfig } from './config' ; import { BinQuery , EncodingQuery , FieldQuery , isAutoCountQuery } from './query/encoding' ; import { ExpandedType } from './query/expandedtype' ; import { cmp , duplicate , extend , keys } from './util' ; const dlBin = dlBin_ ; /**
 * Table Schema Field Descriptor interface
 * see: https://specs.frictionlessdata.io/table-schema/
 */ export interface TableSchemaFieldDescriptor { name : string ; title ? : string ; type : PrimitiveType ; format ? : string ; description ? : string ; } /**
 * Field Schema
 */ export interface FieldSchema extends TableSchemaFieldDescriptor { vlType ? : ExpandedType ; index ? : number ; originalIndex ? : number ; stats : DLFieldProfile ; binStats ? : { [ maxbins ] : DLFieldProfile ; } ; timeStats ? : { [ timeUnit ] : DLFieldProfile ; } ; ordinalDomain ? : string [ ] ; } /**
 * Table Schema
 * see: https://specs.frictionlessdata.io/table-schema/
 */ export interface TableSchema < F extends TableSchemaFieldDescriptor > { fields : F [ ] ; missingValues ? : string [ ] ; primaryKey ? : string | string [ ] ; foreignKeys ? : object [ ] ; } param data return export function build ( data , opt = { } , tableSchema = { fields : [ ] } ) { opt = extend ( { } , DEFAULT_QUERY_CONFIG , opt ) ; const summaries = summary ( data ) ; const types = inferAll ( data ) ; const tableSchemaFieldIndex = tableSchema . fields . reduce ( ( m , field ) => { m [ field . name ] = field ; return m ; } , { } ) ; const fieldSchemas = summaries . map ( function ( fieldProfile , index ) { const name = fieldProfile . field ; const type = types [ name ] === 'date' ? PrimitiveType . DATETIME : ( types [ name ] as any ) ; const distinct = fieldProfile . distinct ; let vlType ; if ( type === PrimitiveType . NUMBER ) { vlType = TYPE . QUANTITATIVE ; } else if ( type === PrimitiveType . INTEGER ) { if ( distinct < opt . numberNominalLimit && distinct / fieldProfile . count < opt . numberNominalProportion ) { vlType = TYPE . NOMINAL ; } else { vlType = TYPE . QUANTITATIVE ; } } else if ( type === PrimitiveType . DATETIME ) { vlType = TYPE . TEMPORAL ; fieldProfile . min = new Date ( data [ 0 ] [ name ] ) ; fieldProfile . max = new Date ( data [ 0 ] [ name ] ) ; for ( const dataEntry of data ) { const time = new Date ( dataEntry [ name ] ) . getTime ( ) ; if ( time < ( fieldProfile . min as Date ) . getTime ( ) ) { fieldProfile . min = new Date ( time ) ; } if ( time > ( fieldProfile . max as Date ) . getTime ( ) ) { fieldProfile . max = new Date ( time ) ; } } } else { vlType = TYPE . NOMINAL ; } if ( vlType === TYPE . NOMINAL && distinct / fieldProfile . count > opt . minPercentUniqueForKey && fieldProfile . count > opt . minCardinalityForKey ) { vlType = ExpandedType . KEY ; } let fieldSchema = { name : name , originalIndex : index , vlType : vlType , type : type , stats : fieldProfile , timeStats : { } as { [ timeUnit ] : DLFieldProfile ; } , binStats : { } as { [ key ] : DLFieldProfile ; } , } ; const orgFieldSchema = tableSchemaFieldIndex [ fieldSchema . name ] ; fieldSchema = extend ( fieldSchema , orgFieldSchema ) ; return fieldSchema ; } ) ; for ( const fieldSchema of fieldSchemas ) { if ( fieldSchema . vlType === TYPE . QUANTITATIVE ) { for ( const maxbins of opt . enum . binProps . maxbins ) { fieldSchema . binStats [ maxbins ] = binSummary ( maxbins , fieldSchema . stats ) ; } } else if ( fieldSchema . vlType === TYPE . TEMPORAL ) { for ( const unit of opt . enum . timeUnit ) { if ( unit !== undefined ) { if ( typeof unit === 'object' ) { if ( 'unit' in unit ) { fieldSchema . timeStats [ unit . unit ] = timeSummary ( unit . unit , fieldSchema . stats ) ; } else { throw new Error ( 'Unrecognized TimeUnit type when calculating fieldSchema.stats' ) ; } } else { fieldSchema . timeStats [ unit ] = timeSummary ( unit , fieldSchema . stats ) ; } } } } } const derivedTableSchema = { ... tableSchema , fields : fieldSchemas , } ; return new Schema ( derivedTableSchema ) ; } const order = { nominal : 0 , key : 1 , ordinal : 2 , temporal : 3 , quantitative : 4 , } ; export class Schema { private _tableSchema : TableSchema < FieldSchema > ; private _fieldSchemaIndex : { [ field ] : FieldSchema ; } ; constructor ( tableSchema ) { this . _tableSchema = tableSchema ; tableSchema . fields . sort ( function ( a , b ) { if ( order [ a . vlType ] < order [ b . vlType ] ) { return - 1 ; } else if ( order [ a . vlType ] > order [ b . vlType ] ) { return 1 ; } else { return a . name . localeCompare ( b . name ) ; } } ) ; tableSchema . fields . forEach ( ( fieldSchema , index ) => ( fieldSchema . index = index ) ) ; this . _fieldSchemaIndex = tableSchema . fields . reduce ( ( m , fieldSchema ) => { m [ fieldSchema . name ] = fieldSchema ; return m ; } , { } ) ; } return public fieldNames ( ) { return this . _tableSchema . fields . map ( ( fieldSchema ) => fieldSchema . name ) ; } return public get fieldSchemas ( ) { return this . _tableSchema . fields ; } public fieldSchema ( fieldName ) { return this . _fieldSchemaIndex [ fieldName ] ; } public tableSchema ( ) { const tableSchema = duplicate ( this . _tableSchema ) ; tableSchema . fields . sort ( ( a , b ) => a . originalIndex - b . originalIndex ) ; return tableSchema ; } return public primitiveType ( fieldName ) { return this . _fieldSchemaIndex [ fieldName ] ? this . _fieldSchemaIndex [ fieldName ] . type : null ; } return public vlType ( fieldName ) { return this . _fieldSchemaIndex [ fieldName ] ? this . _fieldSchemaIndex [ fieldName ] . vlType : null ; } return param augmentTimeUnitDomain public cardinality ( fieldQ , augmentTimeUnitDomain = true , excludeInvalid = false ) { const fieldSchema = this . _fieldSchemaIndex [ fieldQ . field as string ] ; if ( fieldQ . aggregate || ( isAutoCountQuery ( fieldQ ) && fieldQ . autoCount ) ) { return 1 ; } else if ( fieldQ . bin ) { let bin ; if ( typeof fieldQ . bin === 'boolean' ) { bin = { maxbins : autoMaxBins ( fieldQ . channel as Channel ) , } ; } else if ( fieldQ . bin === '?' ) { bin = { enum : [ true , false ] , } ; } else { bin = fieldQ . bin ; } const maxbins = bin . maxbins ; if ( ! fieldSchema . binStats [ maxbins ] ) { fieldSchema . binStats [ maxbins ] = binSummary ( maxbins , fieldSchema . stats ) ; } return fieldSchema . binStats [ maxbins ] . distinct ; } else if ( fieldQ . timeUnit ) { if ( augmentTimeUnitDomain ) { switch ( fieldQ . timeUnit ) { case vegaTime . SECONDS : return 60 ; case vegaTime . MINUTES : return 60 ; case vegaTime . HOURS : return 24 ; case vegaTime . DAY : return 7 ; case vegaTime . DATE : return 31 ; case vegaTime . MONTH : return 12 ; case vegaTime . QUARTER : return 4 ; case vegaTime . MILLISECONDS : return 1000 ; } } const unit = fieldQ . timeUnit as string ; let timeStats = fieldSchema . timeStats ; if ( ! timeStats || ! timeStats [ unit ] ) { timeStats = { ... timeStats , [ unit ] : timeSummary ( fieldQ . timeUnit as TimeUnit , fieldSchema . stats ) , } ; } if ( excludeInvalid ) { return timeStats [ unit ] . distinct - invalidCount ( timeStats [ unit ] . unique , [ 'Invalid Date' , null ] ) ; } else { return timeStats [ unit ] . distinct ; } } else { if ( fieldSchema ) { if ( excludeInvalid ) { return fieldSchema . stats . distinct - invalidCount ( fieldSchema . stats . unique , [ NaN , null ] ) ; } else { return fieldSchema . stats . distinct ; } } else { return null ; } } } /**
     * Given an EncodingQuery with a timeUnit, returns true if the date field
     * has multiple distinct values for all parts of the timeUnit. Returns undefined
     * if the timeUnit is undefined.
     * i.e.
     * ('yearmonth', [Jan 1 2000, Feb 2 2000] returns false)
     * ('yearmonth', [Jan 1 2000, Feb 2 2001] returns true)
     */ public timeUnitHasVariation ( fieldQ ) { if ( ! fieldQ . timeUnit ) { return ; } if ( fieldQ . timeUnit === vegaTime . DAY ) { const dateEncQ = extend ( { } , fieldQ , { timeUnit : vegaTime . DATE } ) ; if ( this . cardinality ( dateEncQ , false , true ) <= 1 ) { return false ; } } const fullTimeUnit = fieldQ . timeUnit ; for ( const timeUnitPart of TIMEUNIT_PARTS ) { if ( containsTimeUnit ( fullTimeUnit as TimeUnit , timeUnitPart ) ) { const singleUnitEncQ = extend ( { } , fieldQ , { timeUnit : timeUnitPart } ) ; if ( this . cardinality ( singleUnitEncQ , false , true ) <= 1 ) { return false ; } } } return true ; } public domain ( fieldQueryParts ) { const fieldSchema = this . _fieldSchemaIndex [ fieldQueryParts . field as string ] ; let domain = keys ( fieldSchema . stats . unique ) ; if ( fieldSchema . vlType === TYPE . QUANTITATIVE ) { return [ + fieldSchema . stats . min , + fieldSchema . stats . max ] ; } else if ( fieldSchema . type === PrimitiveType . DATETIME ) { return [ fieldSchema . stats . min , fieldSchema . stats . max ] ; } else if ( fieldSchema . type === PrimitiveType . INTEGER || fieldSchema . type === PrimitiveType . NUMBER ) { domain = domain . map ( ( x ) => + x ) ; return domain . sort ( cmp ) ; } else if ( fieldSchema . vlType === TYPE . ORDINAL && fieldSchema . ordinalDomain ) { return fieldSchema . ordinalDomain ; } return domain . map ( ( x ) => { return x === 'null' ? null : x ; } ) . sort ( cmp ) ; } return public stats ( fieldQ ) { const fieldSchema = this . _fieldSchemaIndex [ fieldQ . field as string ] ; return fieldSchema ? fieldSchema . stats : null ; } } return function binSummary ( maxbins , summary ) { const bin = dlBin ( { min : summary . min , max : summary . max , maxbins : maxbins , } ) ; const result = extend ( { } , summary ) ; result . unique = binUnique ( bin , summary . unique ) ; result . distinct = ( bin . stop - bin . start ) / bin . step ; result . min = bin . start ; result . max = bin . stop ; return result ; } const SET_DATE_METHOD = { year : 'setFullYear' , month : 'setMonth' , date : 'setDate' , hours : 'setHours' , minutes : 'setMinutes' , seconds : 'setSeconds' , milliseconds : 'setMilliseconds' , dayofyear : null , week : null , quarter : null , day : null , } ; function dateMethods ( singleUnit , isUtc ) { const rawSetDateMethod = SET_DATE_METHOD [ singleUnit ] ; const setDateMethod = isUtc ? `setUTC${ rawSetDateMethod . substr ( 3 ) }` : rawSetDateMethod ; const getDateMethod = `get${ isUtc ? 'UTC' : '' }${ rawSetDateMethod . substr ( 3 ) }` ; return { setDateMethod , getDateMethod } ; } function convert ( unit , date ) { const isUTC = isUTCTimeUnit ( unit ) ; const result = isUTC ? new Date ( Date . UTC ( 1972 , 0 , 1 , 0 , 0 , 0 , 0 ) ) : new Date ( 1972 , 0 , 1 , 0 , 0 , 0 , 0 ) ; for ( const timeUnitPart of TIMEUNIT_PARTS ) { if ( containsTimeUnit ( unit , timeUnitPart ) ) { switch ( timeUnitPart ) { case vegaTime . DAY : throw new Error ( "Cannot convert to TimeUnits containing 'day'" ) ; case vegaTime . DAYOFYEAR : throw new Error ( "Cannot convert to TimeUnits containing 'dayofyear'" ) ; case vegaTime . WEEK : throw new Error ( "Cannot convert to TimeUnits containing 'week'" ) ; case vegaTime . QUARTER : { const { getDateMethod , setDateMethod } = dateMethods ( 'month' , isUTC ) ; result [ setDateMethod ] ( Math . floor ( date [ getDateMethod ] ( ) / 3 ) * 3 ) ; break ; } default : { const { getDateMethod , setDateMethod } = dateMethods ( timeUnitPart , isUTC ) ; result [ setDateMethod ] ( date [ getDateMethod ] ( ) ) ; } } } } return result ; } return function timeSummary ( timeunit , summary ) { const result = extend ( { } , summary ) ; const unique = { } ; keys ( summary . unique ) . forEach ( function ( dateString ) { const date = dateString === 'null' ? null : new Date ( dateString ) ; let key ; if ( date === null ) { key = null ; } else if ( isNaN ( date . getTime ( ) ) ) { key = 'Invalid Date' ; } else { key = ( timeunit === vegaTime . DAY ? date . getDay ( ) : convert ( timeunit , date ) ) . toString ( ) ; } unique [ key ] = ( unique [ key ] || 0 ) + summary . unique [ dateString ] ; } ) ; result . unique = unique ; result . distinct = keys ( unique ) . length ; return result ; } return function binUnique ( bin , oldUnique ) { const newUnique = { } ; for ( const value in oldUnique ) { let bucket ; if ( value === null ) { bucket = null ; } else if ( isNaN ( Number ( value ) ) ) { bucket = NaN ; } else { bucket = bin . value ( Number ( value ) ) as number ; } newUnique [ bucket ] = ( newUnique [ bucket ] || 0 ) + oldUnique [ value ] ; } return newUnique ; } return function invalidCount ( unique , list ) { return list . reduce ( function ( prev , cur ) { return unique [ cur ] ? prev + 1 : prev ; } , 0 ) ; } export enum PrimitiveType { STRING = 'string' as any , NUMBER = 'number' as any , INTEGER = 'integer' as any , BOOLEAN = 'boolean' as any , DATETIME = 'datetime' as any }