import cheerio , { Cheerio , CheerioAPI , Element } from 'cheerio' ; import parseFrontMatter from 'front-matter' ; import fs from 'fs-extra' ; import { last , pickBy } from 'lodash' ; import { resolve } from 'path' ; import { HTMLAttributeReferrerPolicy , LinkHTMLAttributes , ScriptHTMLAttributes , } from 'react' ; import { JupyterBookState , PageFrontMatterData , TOCHeader , } from '@/context/jupyterBook' ; import { getHTMLFiles } from './jupyterBook' ; import { isExternalUrl } from './url' ; interface StackItem < T > { level : number ; node : T ; parentId ? : string ; } interface LinkData { href : string ; text : string ; } interface GetGlobalHeadersOptions < T > { /**
     * Gets the initial list of TOC tree nodes to parse.
     */ getRootNodes ( ) : T [ ] ; param parentNode getLinkData ( parentNode ) : LinkData ; param parentNode getNextNodes ( parentNode , level ) : T [ ] ; } /**
 * Utility function for getting global header data from a DOM tree. The function
 * is implemented generically enough so that it can be used on the browser and
 * by the pre-renderer. Since the DOM is a tree, we can use Depth First Search
 * to traverse the tree and deserialize it into the global headers data
 * structures.
 */ function getGlobalHeaders < T > ( { getRootNodes , getLinkData , getNextNodes , } ) { const globalHeaders = { } ; const rootGlobalHeaders = [ ] ; const stack = getRootNodes ( ) . map ( ( node ) => ( { node , level : 1 , } ) ) ; while ( stack . length > 0 ) { const { level , node , parentId } = stack . pop ( ) ! ; const link = getLinkData ( node ) ; if ( link . href && link . text ) { globalHeaders [ link . href ] = { level , children : [ ] , href : link . href , text : link . text , } ; if ( level === 1 ) { rootGlobalHeaders . push ( link . href ) ; } if ( parentId ) { globalHeaders [ parentId ] . children ?. push ( link . href ) ; } const nextLevel = level + 1 ; const nextNodes = getNextNodes ( node , nextLevel ) ; for ( const nextNode of nextNodes ) { stack . push ( { parentId : link . href , level : level + 1 , node : nextNode , } ) ; } } } rootGlobalHeaders . reverse ( ) ; for ( const header of Object . values ( globalHeaders ) ) { header . children ?. reverse ( ) ; } return { globalHeaders , rootGlobalHeaders , } ; } function getPageHeaders ( $ ) { return $ ( '#page-toc' ) . find ( '#page-toc > ul > li > ul > li > a' ) . toArray ( ) . map < TOCHeader > ( ( element ) => { const link = cheerio ( element ) ; const href = link . attr ( 'href' ) ?? '' ; return { href , text : link . text ( ) , } ; } ) ; } function getTocSelector ( level = 1 ) { return `.toctree-l${ level }` ; } function getGlobalTocHeaders ( globalToc ) { return getGlobalHeaders < Element > ( { getRootNodes : ( ) => globalToc . find ( getTocSelector ( ) ) . toArray ( ) , getLinkData : ( node ) => { const link = cheerio ( node ) . find ( 'a' ) . first ( ) ; let href = link . attr ( 'href' ) ?? '' ; if ( ! isExternalUrl ( href ) ) { href = `/${ href }` ; } return { href , text : link . text ( ) , } ; } , getNextNodes : ( node , level ) => cheerio ( node ) . find ( getTocSelector ( level ) ) . toArray ( ) , } ) ; } /**
 * Scripts that shouldn't be included in the React application.
 */ const IGNORED_APP_SCRIPTS = [ '_static/sphinx-book-theme' ] ; param $ returns function getAppScripts ( $ , selector ) { return ( $ ( selector ) . toArray ( ) . map ( ( script ) => cheerio ( script ) ) . filter ( ( script ) => { const src = script . attr ( 'src' ) ; if ( ! src ) { return true ; } return IGNORED_APP_SCRIPTS . every ( ( file ) => ! src . includes ( file ) ) ; } ) . map ( ( script ) => { const props = { async : script . attr ( 'async' ) === 'true' ? true : undefined , crossOrigin : script . attr ( 'crossorigin' ) , defer : script . attr ( 'defer' ) === 'true' ? true : undefined , integrity : script . attr ( 'integrity' ) , noModule : script . attr ( 'nomodule' ) === 'true' ? true : undefined , nonce : script . attr ( 'nonce' ) , referrerPolicy : script . attr ( 'referrerpolicy' ) as HTMLAttributeReferrerPolicy | undefined , src : script . attr ( 'src' ) , type : script . attr ( 'type' ) , } ; if ( props . src && ! isExternalUrl ( props . src ) ) { props . src = `/_static/${ props . src . replace ( /^.*_static\// , '' ) }` ; } const content = script . html ( ) ; if ( content ) { props . children = content ; } return pickBy ( props , ( value ) => value !== undefined ) ; } ) ) ; } /**
 * Stylesheets that should not be added in the React application.
 */ const IGNORED_STYLE_SHEETS = [ '_static/basic.css' , '_static/pygments_dark.css' ] ; param $ returns function getAppStyleSheets ( $ ) { return ( $ ( 'link[rel=stylesheet]' ) . toArray ( ) . map ( ( linkElement ) => cheerio ( linkElement ) ) . filter ( ( link ) => { const href = link . attr ( 'href' ) ; return ( href && IGNORED_STYLE_SHEETS . every ( ( file ) => ! href . includes ( file ) ) ) ; } ) . map ( ( link ) => { const props = { href : link . attr ( 'href' ) , media : link . attr ( 'media' ) , rel : link . attr ( 'rel' ) , type : link . attr ( 'type' ) , } ; if ( props . href && ! isExternalUrl ( props . href ) ) { props . href = `/_static/${ props . href . replace ( /^.*_static\// , '' ) }` ; } return pickBy ( props , ( value ) => value !== undefined ) ; } ) ) ; } const SEARCH_PAGE_SELECTOR_REMOVE_LIST = [ '.footer' , '.related' , '.sphinxsidebar' , 'h1#search-documentation' , '.body > p:nth-child(2)' , '#fallback > script' , ] ; const ROOT_DIR = resolve ( __dirname , '../../..' ) ; const BUILD_DIR = resolve ( ROOT_DIR , '_build/html' ) ; interface RawFrontMatterData { theme ? : PageFrontMatterData ; } function getMarkdownFile ( file ) { return file . replace ( `${ BUILD_DIR }/` , '' ) . replace ( '.html' , '.md' ) ; } async function getPageFrontMatter ( file ) { const markdownFile = resolve ( ROOT_DIR , getMarkdownFile ( file ) ) ; if ( await fs . pathExists ( markdownFile ) ) { const mdData = await fs . readFile ( markdownFile , 'utf-8' ) ; return parseFrontMatter < RawFrontMatterData > ( mdData ) . attributes . theme ?? { } ; } return { } ; } /**
 * List of images by alt name to exclude from the preview image. This uses the
 * `alt` attribute to determine if the image should be filtered out or not.
 */ const PREVIEW_IMAGE_BLOCKLIST_SET = new Set ( [ 'image.sc forum' , 'License' , 'Build Status' , 'codecov' , 'Python Version' , 'PyPI' , 'PyPI - Downloads' , 'Development Status' , 'Code style: black' , 'DOI' , 'CZI logo' , ] ) ; param pageBody returns function getPreviewImage ( pageBody ) { const [ previewImage ] = pageBody . find ( 'img' ) . toArray ( ) . map ( ( node ) => cheerio ( node ) ) . filter ( ( image ) => ! PREVIEW_IMAGE_BLOCKLIST_SET . has ( image . attr ( 'alt' ) ?? '' ) ) ; return previewImage ?. attr ( 'src' ) ?? '' ; } function getPreviewDescription ( pageBody ) { return pageBody . find ( 'p' ) . first ( ) . text ( ) ; } const MAX_PREVIEW_DESCRIPTION = 200 ; function getIpynbSource ( $ ) { const node = $ ( '#ipynb-source' ) ; const url = node . attr ( 'data-source' ) ; if ( ! url ) { return '' ; } return `/${ url . replaceAll ( /^(\.\.\/)*/g , '' ) }` ; } async function getMdSource ( file ) { const prefix = resolve ( ROOT_DIR , 'public' ) ; const mdFile = getMarkdownFile ( file ) ; const sourceLink = `/_sources/${ mdFile }` ; const localFile = resolve ( prefix , '_sources' , mdFile ) ; if ( await fs . pathExists ( localFile ) ) { return sourceLink ; } return '' ; } param file export async function getPageData ( file ) { const indexFile = resolve ( BUILD_DIR , 'index.html' ) ; const indexHtml = await fs . readFile ( indexFile , 'utf-8' ) ; const globalToc = cheerio . load ( indexHtml ) ( '#global-toc' ) ; const rawHtml = await fs . readFile ( file , 'utf-8' ) ; const $ = cheerio . load ( rawHtml ) ; let result ; const isSearch = last ( file . split ( '/' ) ) === 'search.html' ; const pageBody = $ ( isSearch ? 'body' : '#page-body' ) ; pageBody . find ( 'a.external' ) . toArray ( ) . map ( ( link ) => $ ( link ) ) . forEach ( ( link ) => { link . attr ( 'target' , '_blank' ) ; link . attr ( 'rel' , 'noreferrer' ) ; } ) ; if ( isSearch ) { SEARCH_PAGE_SELECTOR_REMOVE_LIST . forEach ( ( selector ) => pageBody . find ( selector ) . remove ( ) ) ; pageBody . find ( 'form' ) . attr ( 'style' , 'display: none' ) ; result = { ... getGlobalTocHeaders ( globalToc ) , pageTitle : 'Search' , appScripts : getAppScripts ( $ , 'head script' ) , appStyleSheets : getAppStyleSheets ( $ ) , pageBodyHtml : pageBody . html ( ) ?? '' , pageHeaders : [ ] , pageFrontMatter : { } , previewImage : '' , previewDescription : '' , ipynbSource : '' , mdSource : '' , } ; } else { const pageHeader = pageBody . find ( 'h1' ) . first ( ) ; const pageTitle = pageHeader . text ( ) . replace ( '¶' , '' ) ; pageHeader . remove ( ) ; const pageFrontMatter = await getPageFrontMatter ( file ) ; let previewImage = pageFrontMatter . previewImage || getPreviewImage ( pageBody ) ; if ( ! previewImage ) { const files = await getHTMLFiles ( ) ; const tutorialsIndexFile = files . find ( ( htmlFile ) => htmlFile . includes ( 'tutorials/index.html' ) ) ; if ( tutorialsIndexFile ) { const tutorialsHtml = await fs . readFile ( tutorialsIndexFile , 'utf-8' ) ; const tutorialsPageBody = cheerio . load ( tutorialsHtml ) ( '#page-body' ) ; previewImage = getPreviewImage ( tutorialsPageBody ) ; } } let previewDescription = pageFrontMatter . metaDescription || pageFrontMatter . intro || getPreviewDescription ( pageBody ) ; if ( previewDescription . length > MAX_PREVIEW_DESCRIPTION ) { previewDescription = previewDescription . slice ( 0 , MAX_PREVIEW_DESCRIPTION ) . replaceAll ( /.{3}$/g , '...' ) ; } pageBody . find ( '.tableofcontents-wrapper' ) . remove ( ) ; pageBody . find ( '.toctree-wrapper' ) . remove ( ) ; result = { ... getGlobalTocHeaders ( globalToc ) , pageTitle , pageFrontMatter , previewImage , previewDescription , appScripts : getAppScripts ( $ , '#scripts script' ) , appStyleSheets : getAppStyleSheets ( $ ) , pageBodyHtml : pageBody . html ( ) ?? '' , pageHeaders : getPageHeaders ( $ ) , ipynbSource : getIpynbSource ( $ ) , mdSource : await getMdSource ( file ) , } ; } return result ; }