import cheerio , { Cheerio , CheerioAPI , Element } from 'cheerio' ; import { getCustomCss } from './custom-css' ; import { contentSelectorRules } from './rules' ; const divToPElementsRegexp = /<(a|blockquote|dl|div|img|ol|p|pre|table|ul)/i ; const blacklistCandidatesRegexp = /popupbody|-ad|g-plus/i ; const unlikelyCandidatesRegexp = /banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|header|legends|menu|modal|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i ; const okMaybeItsACandidateRegexp = /and|article|body|column|main|shadow/i ; const negativeRegexp = /hidden|^hid$|hid$|hid|^hid |banner|combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|modal|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget|byline|author|dateline|writtenby|p-author/i ; const positiveRegexp = /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i ; const sentenceRegexp = /\.( |$)/ ; const defaultTagsToScore = 'section,h2,h3,h4,h5,h6,p,td,pre,div' ; export function extractContent ( hostname , html ) { const $ = cheerio . load ( html ) ; $ ( 'script, style' ) . each ( function ( ) { $ ( this ) . remove ( ) ; } ) ; const contentSelector = contentSelectorRules [ hostname ] || 'body' ; const $content = cheerio . load ( $ ( contentSelector ) . html ( ) ) ; transformMisusedDivsIntoParagraphs ( $content ) ; removeUnlikelyCandidates ( $content ) ; const candidates = getCandidates ( $content ) ; const topCandidate = getTopCandidate ( $content , candidates ) ; let output = getArticle ( $content , topCandidate , candidates ) ; const title = $ ( 'meta[property="og:title"]' ) . attr ( 'content' ) || $ ( 'title' ) . text ( ) ; const description = $ ( 'meta[property="og:description"]' ) . attr ( 'content' ) || $ ( 'meta[name="description"]' ) . attr ( 'content' ) ; const css = getCustomCss ( $ , hostname ) ; if ( css ) { output += `<style>${ css }</style>` ; } return { title , description , content : output , } ; } function transformMisusedDivsIntoParagraphs ( $ ) { $ ( 'div' ) . each ( function ( ) { const html = $ ( this ) . html ( ) ; if ( ! divToPElementsRegexp . test ( html ) ) { $ ( this ) . get ( 0 ) . tagName = 'p' ; } } ) ; } function removeUnlikelyCandidates ( $ ) { $ ( '*' ) . not ( 'html,body' ) . each ( function ( ) { const className = $ ( this ) . attr ( 'class' ) ; const id = $ ( this ) . attr ( 'id' ) ; const str = className + id ; if ( blacklistCandidatesRegexp . test ( str ) || ( unlikelyCandidatesRegexp . test ( str ) && ! okMaybeItsACandidateRegexp . test ( str ) ) ) { $ ( this ) . remove ( ) ; } } ) ; } type Candidate = { el : Cheerio < Element > ; score : number ; } ; type Candidates = Map < Element , Candidate > ; function getCandidates ( $ ) { const candidates = new Map ( ) ; $ ( defaultTagsToScore ) . each ( function ( ) { const el = $ ( this ) ; const text = el . text ( ) ; if ( text . length < 25 ) { return ; } const parent = el . parent ( ) ; const parentNode = parent . get ( 0 ) ; const grandParent = parent . parent ( ) ; let grandParentNode ; if ( grandParent . length > 0 ) { grandParentNode = grandParent . get ( 0 ) ; } const found = candidates . get ( parentNode ) ; if ( ! found ) { candidates . set ( parentNode , scoreNode ( parent ) ) ; } if ( grandParentNode != null ) { const found = candidates . get ( grandParentNode ) ; if ( ! found ) { candidates . set ( grandParentNode , scoreNode ( grandParent ) ) ; } } let contentScore = 1 ; contentScore += ( text . match ( /,/g ) || [ ] ) . length ; contentScore += Math . min ( text . length / 100 , 3 ) ; candidates . get ( parentNode ) . score += contentScore ; if ( grandParentNode != null ) { candidates . get ( grandParentNode ) . score += contentScore / 2 ; } } ) ; for ( const c of candidates . values ( ) ) { c . score = c . score * ( 1 - getLinkDensity ( c . el ) ) ; } return candidates ; } function scoreNode ( el ) { const c = { el , score : 0 } ; switch ( el . get ( 0 ) . tagName . toLowerCase ( ) ) { case 'div' : c . score += 5 ; case 'pre' : case 'td' : case 'blockquote' : case 'img' : c . score += 3 ; case 'address' : case 'ol' : case 'ul' : case 'dl' : case 'dd' : case 'dt' : case 'li' : case 'form' : c . score -= 3 ; case 'h1' : case 'h2' : case 'h3' : case 'h4' : case 'h5' : case 'h6' : case 'th' : c . score -= 5 ; } c . score += getClassWeight ( el ) ; return c ; } function getClassWeight ( el ) { let weight = 0 ; const className = el . attr ( 'class' ) ; const id = el . attr ( 'id' ) ; if ( className ) { if ( negativeRegexp . test ( className ) ) { weight -= 25 ; } if ( positiveRegexp . test ( className ) ) { weight += 25 ; } } if ( id ) { if ( negativeRegexp . test ( id ) ) { weight -= 25 ; } if ( positiveRegexp . test ( id ) ) { weight += 25 ; } } return weight ; } function getLinkDensity ( el ) { const linkLength = el . find ( 'a' ) . text ( ) . length ; const textLength = el . text ( ) . length ; if ( textLength === 0 ) { return 0 ; } return linkLength / textLength ; } function getTopCandidate ( $ , candidates ) { let best ; for ( const c of candidates . values ( ) ) { if ( best == null ) { best = c ; } else if ( best . score < c . score ) { best = c ; } } if ( best == null ) { best = { el : $ ( 'body' ) , score : 0 } ; } return best ; } function getArticle ( $ , topCandidate , candidates ) { let output = '<!--start--><div>' ; const siblingScoreThreshold = Math . max ( 10 , topCandidate . score * 0.2 ) ; topCandidate . el . siblings ( ) . add ( topCandidate . el ) . each ( function ( ) { const el = $ ( this ) ; let append = false ; let node = el . get ( 0 ) ; if ( node === topCandidate . el . get ( 0 ) ) { append = true ; } else if ( candidates . has ( node ) && candidates . get ( node ) . score >= siblingScoreThreshold ) { append = true ; } if ( el . is ( 'p' ) ) { const linkDensity = getLinkDensity ( el ) ; const content = el . text ( ) ; const contentLength = content . length ; if ( contentLength >= 80 && linkDensity < 0.25 ) { append = true ; } else if ( contentLength < 80 && linkDensity == 0 && sentenceRegexp . test ( content ) ) { append = true ; } } if ( append ) { let tag = 'div' ; if ( el . is ( 'p' ) ) { tag = node . tagName ; } const html = el . html ( ) ; output += `<!--append-start--><${ tag }>${ html }</${ tag }><!--append-end-->` ; } } ) ; output += '</div><!--end-->' ; return output ; }