fileoverview import TabularDataset , { TabularRows , TabularColumn } from '@/data/tabular-dataset' ; import { dsvFormat } from 'd3-dsv' ; import _ from 'lodash' ; import { isNumericalType } from '@/data/util' ; /** Value types ordered from the strickest to weakest typing */ export enum ValueType { EMPTY = 'empty' , DATE = 'date' , INT = 'int' , FLOAT = 'float' , STRING = 'string' , ERROR = 'error' } const ValueTypeOrder = { empty : 0 , date : 1 , int : 2 , float : 3 , string : 4 , error : 5 } ; const isNumber = ( text ) : boolean => { return Number ( text ) === + text ; } ; const isLooserType = ( a , b ) : boolean => { return ValueTypeOrder [ a ] > ValueTypeOrder [ b ] ; } ; const VALID_DATE_START = Date . parse ( '1000 GMT' ) ; const VALID_DATE_END = Date . parse ( '2200 GMT' ) ; export const isProbablyTimestamp = ( text ) : boolean => { return isNumber ( text ) && text . match ( /^-?[0-9]+$/ ) !== null && VALID_DATE_START <= + text && + text <= VALID_DATE_END && text . length >= 9 ; } ; export const isProbablyDate = ( text ) : boolean => { if ( text === null || text === undefined ) { return false ; } if ( isNumber ( text ) ) { if ( text . match ( /^[0-9]{4,4}$/ ) && 1000 <= + text && + text <= 2200 ) { return true ; } if ( isProbablyTimestamp ( text ) ) { return true ; } return false ; } const date = new Date ( text ) ; return date . toString ( ) !== 'Invalid Date' && date . getTime ( ) >= VALID_DATE_START && date . getTime ( ) <= VALID_DATE_END ; } ; /** Parses a token and returns its value and type. */ const checkToken = ( text , ignoredTypes_ ? ) : { type : ValueType ; value : number | string ; } => { const ignoredTypes = new Set < ValueType > ( ignoredTypes_ ? ignoredTypes_ : [ ] ) ; text = text + '' ; if ( ! ignoredTypes . has ( ValueType . EMPTY ) && text === '' ) { return { type : ValueType . EMPTY , value : '' } ; } if ( ! ignoredTypes . has ( ValueType . DATE ) && isProbablyDate ( text ) ) { return { type : ValueType . DATE , value : new Date ( text ) . getTime ( ) } ; } const res = text . match ( /^-?[0-9]+/ ) ; if ( ! ignoredTypes . has ( ValueType . INT ) && res && res [ 0 ] === text ) { return { type : ValueType . INT , value : parseInt ( text , 10 ) } ; } if ( ! ignoredTypes . has ( ValueType . FLOAT ) && isNumber ( text ) ) { return { type : ValueType . FLOAT , value : parseFloat ( text ) } ; } if ( ! ignoredTypes . has ( ValueType . STRING ) ) { return { type : ValueType . STRING , value : text } ; } return { type : ValueType . ERROR , value : '' } ; } ; /** Tokenizes the input value to form a value of chosen type. */ export const parseToken = ( text , type ? ) : number | string => { type = ! type ? checkToken ( text ) . type : type ; if ( text === '' ) { switch ( type ) { case ValueType . INT : case ValueType . FLOAT : case ValueType . DATE : return 0 ; default : return '' ; } } switch ( type ) { case ValueType . INT : return parseInt ( text , 10 ) ; case ValueType . FLOAT : return parseFloat ( text ) ; case ValueType . DATE : return text ; case ValueType . STRING : return '' + text ; default : return '' ; } } ; /** Parses a column and returns its type. */ const checkColumnType = ( rows , columnIndex , name ) : TabularColumn => { let columnType = ValueType . EMPTY ; rows . forEach ( row => { const type = checkToken ( row [ columnIndex ] ) . type ; if ( isLooserType ( type , columnType ) ) { columnType = type ; } } ) ; let hasDuplicate = false ; const existing = new Set ( ) ; for ( const row of rows ) { const value = parseToken ( row [ columnIndex ] . toString ( ) , columnType ) ; row [ columnIndex ] = value ; if ( existing . has ( value ) ) { hasDuplicate = true ; break ; } else { existing . add ( value ) ; } } return { index : columnIndex , name , type : columnType , hasDuplicate } ; } ; const formatColumnType = ( rows , column ) => { for ( const row of rows ) { let value = row [ column . index ] ; if ( column . type === ValueType . DATE ) { value = value . toString ( ) ; } else if ( isNumericalType ( column . type ) ) { value = + value ; } else { value = value . toString ( ) ; } row [ column . index ] = value ; } } ; /** Parses a CSV string and generates its TabularDataset. */ export const parseCsv = ( csv ) : TabularDataset => { let headerLine ; const firstNewLine = csv . indexOf ( '\n' ) ; if ( firstNewLine === - 1 ) { headerLine = csv ; } else { headerLine = csv . substr ( 0 , firstNewLine ) ; } let delimiter = ',' ; const delimiterMatched = headerLine . match ( /[,;|]/ ) ; if ( delimiterMatched !== null ) { delimiter = delimiterMatched [ 0 ] ; } let rows = dsvFormat ( delimiter ) . parseRows ( csv ) ; const columnNames = rows . splice ( 0 , 1 ) [ 0 ] ; if ( _ . last ( columnNames ) === '' ) { columnNames . pop ( ) ; rows = rows . map ( row => row . slice ( 0 , columnNames . length ) ) ; } for ( const row of rows ) { if ( ! ( row instanceof Array ) || row . length < columnNames . length ) { throw new Error ( 'data is not valid CSV' ) ; } } const columns = columnNames . map ( ( name , columnIndex ) => checkColumnType ( rows , columnIndex , name ) ) ; columns . forEach ( column => formatColumnType ( rows , column ) ) ; return new TabularDataset ( { columns , rows } ) ; } ; /**
 * Produces a text CSV from a list of columns and rows.
 */ export const generateCsv = ( columns , rows ) : string => { let result = columns . join ( ',' ) + '\n' ; for ( const row of rows ) { result += row . join ( ',' ) + '\n' ; } return result ; } ;