import { sampleStdev } from 'stats-lite' ; import { iteratorNode } from '../utils' ; export default class ContentExtractor { contentTag : string ; nodeInfo : any [ ] ; punctuation : Set < string > ; constructor ( contentTag = 'p' ) { this . contentTag = contentTag ; this . nodeInfo = [ ] ; this . punctuation = new Set ( [ ... `！，。？、；：“”‘’《》%（）,.?:;'"!%()` ] ) ; } extract ( $ ) { $ ( 'body *' ) . each ( ( idx , node ) => { const { density , tiText , ti , lti , tgi , ltgi } = this . calcTextDensity ( node , $ ) ; const textTagCount = this . countTextTag ( $ , node , this . contentTag ) ; const sbdi = this . calcSbdi ( tiText , ti , lti ) ; const info = { ti , lti , tgi , ltgi , node , density , text : tiText , textTagCount , sbdi } ; this . nodeInfo . push ( info ) ; } ) ; const std = this . calcStandardDeviation ( ) ; this . calcNewScore ( std ) ; const result = this . nodeInfo . sort ( ( a , b ) => b . score - a . score ) ; return result ; } countTextTag ( $ , element , tag = 'p' ) { return $ ( element ) . find ( tag ) . length ; } getAllTiText ( element , $ ) { const textList = [ ] ; for ( const node of iteratorNode ( element ) ) { if ( node . type === 'text' ) { const text = $ ( node ) . text ( ) . replace ( /[\n\r]/g , ' ' ) . replace ( /\s{1,}/g , ' ' ) . trim ( ) ; text && textList . push ( text ) ; } } return textList ; } getAllLtiText ( element , $ ) { const textList = [ ] ; for ( const node of iteratorNode ( element ) ) { if ( node . tagName === 'a' ) { const text = $ ( node ) . text ( ) . replace ( /[\n\r]/g , ' ' ) . replace ( /\s{1,}/g , ' ' ) . trim ( ) ; text && textList . push ( text ) ; } } return textList ; } calcNewScore ( std ) { for ( const info of this . nodeInfo ) { const score = Math . log ( std ) * info . density * Math . log10 ( info . textTagCount + 2 ) * Math . log ( info . sbdi ) ; info . score = isNaN ( score ) ? 0 : score ; } } calcStandardDeviation ( ) { const scoreList = this . nodeInfo . map ( item => item . density ) ; const std = sampleStdev ( scoreList ) ; return std ; } calcTextDensity ( element , $ ) { const tiText = this . getAllTiText ( element , $ ) . join ( '\n' ) ; const ti = tiText . length ; const lti = this . getAllLtiText ( element , $ ) . join ( '\n' ) . length ; const tgi = $ ( element ) . find ( '*' ) . length ; const ltgi = $ ( element ) . find ( 'a' ) . length ; let density = 0 ; if ( tgi - ltgi !== 0 ) { density = ( ti - lti ) / ( tgi - ltgi ) ; } return { density , tiText , ti , lti , tgi , ltgi } ; } countPunctuationNum ( text ) { let count = 0 ; for ( const char of text ) { if ( this . punctuation . has ( char ) ) { count += 1 ; } } return count ; } calcSbdi ( text , ti , lti ) { const sbi = this . countPunctuationNum ( text ) ; const sbdi = ( ti - lti ) / ( sbi + 1 ) ; return sbdi || 1 ; } }