import cheerio from 'cheerio' ; import { cli } from 'cli-ux' ; import { format } from 'date-fns' ; import { copyFileSync , existsSync , lstatSync , mkdirSync , rmdirSync , readdirSync , readFileSync , renameSync , closeSync , openSync , opendirSync , unlinkSync , writeFileSync } from 'fs' ; import Handlebars from 'handlebars' ; import fetch from 'node-fetch' ; import path from 'path' ; import { join , basename , relative } from 'path' ; import { appendHtmlPostfix , makeScriptLinksRelativeToWiki , moveRelativeLinksUpOneLevel , moveRelativeLinksDownOneLevel , prefixRelativeRoot , reworkLinks , reworkScriptSrcs , reworkRedirect } from './article-transforms' ; import { Directories , Options } from './domain' ; import { downloadFile } from './utils/download-file' ; const indexRedirectFragment = readFileSync ( './src/templates/index_redirect_fragment.handlebars' ) ; const footerFragment = readFileSync ( './src/templates/footer_fragment.handlebars' ) ; export const copyImageAssetsIntoWiki = async ( assetsDir , { imagesFolder } ) => { cli . action . start ( '  Copying image assets into unpacked zim directory ' ) ; const imagesFiles = readdirSync ( assetsDir ) ; for ( const imageFile of imagesFiles ) { const filepath = join ( assetsDir , imageFile ) ; const info = lstatSync ( filepath ) ; if ( ! info . isFile ( ) ) { return ; } const imagesFolderPath = join ( imagesFolder , imageFile ) ; copyFileSync ( filepath , imagesFolderPath ) ; } cli . action . stop ( ) ; } ; export const moveArticleFolderToWiki = ( { articleFolder , wikiFolder } ) => { if ( existsSync ( wikiFolder ) ) { return ; } cli . action . start ( '  Renaming A namespace to wiki ' ) ; renameSync ( articleFolder , wikiFolder ) ; cli . action . stop ( ) ; } ; export const fixFavicon = ( { unpackedZimDir } ) => { const favicon = join ( unpackedZimDir , '-' , 'favicon' ) ; const faviconIco = join ( unpackedZimDir , 'favicon.ico' ) ; if ( existsSync ( faviconIco ) || ! existsSync ( favicon ) ) { return ; } cli . action . start ( '  Fixing favicon ' ) ; copyFileSync ( favicon , faviconIco ) ; cli . action . stop ( ) ; } ; export const fixExceptions = async ( { unpackedZimDir , wikiFolder } ) => { const exceptionsDir = join ( unpackedZimDir , '_exceptions' ) ; if ( ! existsSync ( exceptionsDir ) ) { return ; } const dir = opendirSync ( exceptionsDir ) ; for await ( let file of dir ) { let articleName ; try { articleName = decodeURIComponent ( file . name ) ; } catch ( e ) { console . error ( `[fixExceptions] unable to decodeURIComponent(${ file . name }), skipping ` ) ; continue ; } const segments = articleName . split ( '/' ) ; if ( segments [ 0 ] !== 'A' ) continue ; segments . shift ( ) ; if ( ! segments . length || segments . some ( s => ! s . length ) ) continue ; const suffixFile = segments . pop ( ) || '' ; const fixRelativeLinks = ( filePath , depth ) => { const fileBytes = readFileSync ( filePath ) ; const $fileHtml = cheerio . load ( fileBytes . toString ( ) ) ; const linkFixups = Array . from ( { length : depth } , ( x , i ) => moveRelativeLinksDownOneLevel ) ; reworkLinks ( $fileHtml , 'a:not(a[href^="http"]):not(a[href^="//"])' , linkFixups ) ; reworkLinks ( $fileHtml , 'link[href^="../"]' , linkFixups ) ; reworkScriptSrcs ( $fileHtml , 'img' , linkFixups ) ; reworkScriptSrcs ( $fileHtml , 'script' , linkFixups ) ; reworkRedirect ( $fileHtml , 'meta[http-equiv="refresh"]' , linkFixups ) ; writeFileSync ( filePath , $fileHtml . html ( ) ) ; } ; if ( segments . length ) { for ( let i = 1 ; i < segments . length + 1 ; i ++ ) { const parentDir = join ( wikiFolder , ... segments . slice ( 0 , i ) ) ; if ( existsSync ( parentDir ) ) { if ( lstatSync ( parentDir ) . isFile ( ) ) { const articleTmp = `${ parentDir }.tmp` ; const articleDst = join ( parentDir , 'index.html' ) ; renameSync ( parentDir , articleTmp ) ; mkdirSync ( parentDir , { recursive : true } ) ; renameSync ( articleTmp , articleDst ) ; fixRelativeLinks ( articleDst , i ) ; } } else { mkdirSync ( parentDir , { recursive : true } ) ; } } } const articleSrc = join ( exceptionsDir , file . name ) ; const articleDir = join ( wikiFolder , ... segments ) ; const articleDst = join ( articleDir , suffixFile ) ; if ( existsSync ( articleDst ) && lstatSync ( articleDst ) . isDirectory ( ) ) { const movedArticleDst = join ( articleDst , 'index.html' ) ; renameSync ( articleSrc , movedArticleDst ) ; fixRelativeLinks ( movedArticleDst , 1 ) ; } else { renameSync ( articleSrc , articleDst ) ; } } } ; export const includeSourceZim = ( { zimFile , unpackedZimDir } ) => { const zimCopy = join ( unpackedZimDir , basename ( zimFile ) ) ; if ( existsSync ( zimCopy ) ) { return ; } cli . action . start ( '  Copying source ZIM to the root of unpacked zim directory ' ) ; copyFileSync ( zimFile , zimCopy ) ; cli . action . stop ( ) ; } ; export const insertIndexRedirect = ( options ) => { cli . action . start ( "  Inserting root 'index.html' as redirect to main page" ) ; const template = Handlebars . compile ( indexRedirectFragment . toString ( ) ) ; const indexPath = join ( options . unpackedZimDir , 'index.html' ) ; const wikiIndexPath = join ( options . unpackedZimDir , 'wiki' , 'index.html' ) ; if ( existsSync ( indexPath ) ) { unlinkSync ( indexPath ) ; } writeFileSync ( indexPath , template ( { MAIN_PAGE : 'wiki/' } ) ) ; if ( existsSync ( wikiIndexPath ) ) { unlinkSync ( wikiIndexPath ) ; } writeFileSync ( wikiIndexPath , template ( { MAIN_PAGE : `./${ options . mainPage }` } ) ) ; cli . action . stop ( ) ; } ; export const resolveDirectories = ( options ) => { const articleFolder = join ( options . unpackedZimDir , 'A' ) ; const imagesFolder = join ( options . unpackedZimDir , 'I' ) ; const wikiFolder = join ( options . unpackedZimDir , 'wiki' ) ; const jsmodulesFolder = join ( options . unpackedZimDir , '-' ) ; const directories = { unpackedZimDir : options . unpackedZimDir , articleFolder , imagesFolder , wikiFolder , jsmodulesFolder } ; return directories ; } ; const unpackedArticlePath = ( wikiFolder , article ) => { let articlePath = join ( wikiFolder , article ) ; if ( ! existsSync ( articlePath ) ) throw new Error ( `unpacked '/wiki/${ article }' is missing` ) ; if ( lstatSync ( articlePath ) . isDirectory ( ) ) { const fixedSrc = join ( articlePath , 'index.html' ) ; if ( existsSync ( fixedSrc ) ) { return fixedSrc ; } else { throw new Error ( `unpacked '/wiki/${ article }' is a dir without index.html` ) ; } } return articlePath ; } ; export const useKiwixLandingPage = async ( options , { wikiFolder , imagesFolder } ) => { cli . action . start ( `  Generating landing page at /wiki/ from Kiwix one at /wiki/${ options . kiwixMainPage }` ) ; const landingPagePath = join ( wikiFolder , 'index.html' ) ; const kiwixMainPageSrc = unpackedArticlePath ( wikiFolder , options . kiwixMainPage ) ; copyFileSync ( kiwixMainPageSrc , landingPagePath ) ; if ( kiwixMainPageSrc . includes ( 'The_other_Kiwix_guy' ) ) { const $landingHtml = cheerio . load ( readFileSync ( landingPagePath ) . toString ( ) ) ; const canonicalUrlString = $landingHtml ( 'link[rel="canonical"]' ) . attr ( 'href' ) ; if ( ! canonicalUrlString ) { throw new Error ( `Could not parse out canonical url for ${ canonicalUrlString }` ) ; } const canonicalUrl = new URL ( canonicalUrlString ) ; canonicalUrl . pathname = `wiki/${ options . mainPage }` ; const response = await fetch ( canonicalUrl ) ; const pageBody = await response . text ( ) ; const $remoteMainPageHtml = cheerio . load ( pageBody ) ; const pageTitle = $remoteMainPageHtml ( 'title' ) . text ( ) ; $landingHtml ( 'title' ) . text ( pageTitle ) ; writeFileSync ( landingPagePath , $landingHtml . html ( ) ) ; } const depth = ( options . kiwixMainPage . match ( /\//g ) || [ ] ) . length ; if ( depth ) { const fixRelativeLinksUp = ( filePath , depth ) => { const fileBytes = readFileSync ( filePath ) ; const $fileHtml = cheerio . load ( fileBytes . toString ( ) ) ; const linkFixups = Array . from ( { length : depth } , ( x , i ) => moveRelativeLinksUpOneLevel ) ; reworkLinks ( $fileHtml , 'a:not(a[href^="http"]):not(a[href^="//"])' , linkFixups ) ; reworkLinks ( $fileHtml , 'link[href^="../"]' , linkFixups ) ; reworkScriptSrcs ( $fileHtml , 'img' , linkFixups ) ; reworkScriptSrcs ( $fileHtml , 'script' , linkFixups ) ; writeFileSync ( filePath , $fileHtml . html ( ) ) ; } ; fixRelativeLinksUp ( landingPagePath , depth ) ; } cli . action . stop ( ) ; } ; export const appendJavscript = ( options , { unpackedZimDir , jsmodulesFolder } ) => { cli . action . start ( '  Appending custom javascript to site.js ' ) ; const delimiter = '/* Appended by Distributed Wikipedia Mirror – details at https://github.com/ipfs/distributed-wikipedia-mirror */' ; const targetSiteJsFile = join ( jsmodulesFolder , 'mw' , 'site.js' ) ; const dwmSitejsTemplate = readFileSync ( './src/templates/site.js.handlebars' ) . toString ( ) . replace ( '<script>' , '' ) . replace ( '</script>' , '' ) ; const context = { SNAPSHOT_DATE : format ( new Date ( ) , 'yyyy-MM' ) , HOSTING_IPNS_HASH : options . hostingIPNSHash , HOSTING_DNS_DOMAIN : options . hostingDNSDomain , ZIM_NAME : basename ( options . zimFile ) } ; const dwmSitejs = Handlebars . compile ( dwmSitejsTemplate . toString ( ) ) ( { FOOTER_TEMPLATE : footerFragment . toString ( ) . replace ( /\n/g , '\\\n' ) . replace ( /"/g , '\\"' ) , DWM_OPTIONS : JSON . stringify ( context , null , 2 ) } ) ; let originalSiteJs = readFileSync ( targetSiteJsFile ) . toString ( ) ; originalSiteJs = originalSiteJs . replace ( 'if(wgCanonicalSpecialPageName=="Watchlist")importScript(\'MediaWiki:Common.js/WatchlistNotice.js\');' , '' ) ; if ( originalSiteJs . includes ( delimiter ) ) { originalSiteJs = originalSiteJs . split ( delimiter ) [ 0 ] ; } const updatedSiteJs = `\n${ delimiter }\n${ dwmSitejs }` ; writeFileSync ( targetSiteJsFile , updatedSiteJs ) ; const targetJsConfigVarFile = join ( jsmodulesFolder , 'mw' , 'jsConfigVars.js' ) ; writeFileSync ( targetJsConfigVarFile , '{}' ) ; const startupJsFile = join ( jsmodulesFolder , 'mw' , 'startup.js' ) ; let startJs = readFileSync ( startupJsFile ) . toString ( ) ; startJs = startJs . replace ( 'function domEval(code){var' , 'function domEval(code){' ) . replace ( '"/w/load.php"' , '"../w/load.php"' ) ; writeFileSync ( startupJsFile , startJs ) ; const loadPhpPath = join ( unpackedZimDir , 'w' , 'load.php' ) ; if ( ! existsSync ( loadPhpPath ) ) { mkdirSync ( join ( unpackedZimDir , 'w' ) ) ; writeFileSync ( loadPhpPath , '/* Stubbed by Distributed Wikipedia Mirror */' ) ; } for ( const file of [ 'ext.cite.ux-enhancements.js' ] ) { const filepath = join ( jsmodulesFolder , 'mw' , file ) ; const overwriteText = '/* Overwritten by Distributed Wikipedia Mirror to prevent js errors, see https://github.com/openzim/mwoffliner/issues/894 */' ; writeFileSync ( filepath , overwriteText ) ; } cli . action . stop ( ) ; } ;