import type { IMessageFromBackground } from '@cliqz/adblocker-content' ; import Config from '../config' ; import { StaticDataView , sizeOfASCII , sizeOfByte } from '../data-view' ; import { EventEmitter } from '../events' ; import { adsAndTrackingLists , adsLists , Fetch , fetchLists , fetchResources , fullLists , } from '../fetch' ; import { HTMLSelector } from '../html-filtering' ; import CosmeticFilter from '../filters/cosmetic' ; import NetworkFilter from '../filters/network' ; import { block } from '../filters/dsl' ; import { IListDiff , IRawDiff , parseFilters } from '../lists' ; import Request from '../request' ; import Resources from '../resources' ; import CosmeticFilterBucket from './bucket/cosmetic' ; import NetworkFilterBucket from './bucket/network' ; export const ENGINE_VERSION = 496 ; function shouldApplyHideException ( filters ) { if ( filters . length === 0 ) { return false ; } let genericHideFilter ; let currentScore = 0 ; for ( const filter of filters ) { const score = ( filter . isImportant ( ) ? 4 : 0 ) | ( filter . isException ( ) ? 1 : 2 ) ; if ( score >= currentScore ) { currentScore = score ; genericHideFilter = filter ; } } if ( genericHideFilter === undefined ) { return false ; } return genericHideFilter . isException ( ) ; } export interface BlockingResponse { match : boolean ; redirect : undefined | { body : string ; contentType : string ; dataUrl : string ; } ; exception : NetworkFilter | undefined ; filter : NetworkFilter | undefined ; } export interface Caching { path : string ; read : ( path : string ) => Promise < Uint8Array > ; write : ( path : string , buffer : Uint8Array ) => Promise < void > ; } export default class FilterEngine extends EventEmitter < 'csp-injected' | 'html-filtered' | 'request-allowed' | 'request-blocked' | 'request-redirected' | 'request-whitelisted' | 'script-injected' | 'style-injected' > { private static fromCached < T extends typeof FilterEngine > ( this , init , caching ? ) { if ( caching === undefined ) { return init ( ) ; } const { path , read , write } = caching ; return read ( path ) . then ( ( buffer ) => this . deserialize ( buffer ) as InstanceType < T > ) . catch ( ( ) => init ( ) . then ( ( engine ) => write ( path , engine . serialize ( ) ) . then ( ( ) => engine ) ) ) ; } public static empty < T extends FilterEngine > ( this , config = { } ) { return new this ( { config } ) ; } /**
     * Create an instance of `FiltersEngine` (or subclass like `ElectronBlocker`,
     * etc.), from the list of subscriptions provided as argument (e.g.:
     * EasyList).
     *
     * Lists are fetched using the instance of `fetch` provided as a first
     * argument. Optionally resources.txt and config can be provided.
     */ public static fromLists < T extends typeof FilterEngine > ( this , fetch , urls , config = { } , caching ? ) { return this . fromCached ( ( ) => { const listsPromises = fetchLists ( fetch , urls ) ; const resourcesPromise = fetchResources ( fetch ) ; return Promise . all ( [ listsPromises , resourcesPromise ] ) . then ( ( [ lists , resources ] ) => { const engine = this . parse ( lists . join ( '\n' ) , config ) ; if ( resources !== undefined ) { engine . updateResources ( resources , '' + resources . length ) ; } return engine as InstanceType < T > ; } ) ; } , caching ) ; } /**
     * Initialize blocker of *ads only*.
     *
     * Attempt to initialize a blocking engine using a pre-built version served
     * from Cliqz's CDN. If this fails (e.g.: if no pre-built engine is available
     * for this version of the library), then falls-back to using `fromLists(...)`
     * method with the same subscriptions.
     */ public static fromPrebuiltAdsOnly < T extends typeof FilterEngine > ( this , fetchImpl = fetch , caching ? ) { return this . fromLists ( fetchImpl , adsLists , { } , caching ) ; } /**
     * Same as `fromPrebuiltAdsOnly(...)` but also contains rules to block
     * tracking (i.e.: using extra lists such as EasyPrivacy and more).
     */ public static fromPrebuiltAdsAndTracking < T extends typeof FilterEngine > ( this , fetchImpl = fetch , caching ? ) { return this . fromLists ( fetchImpl , adsAndTrackingLists , { } , caching ) ; } /**
     * Same as `fromPrebuiltAdsAndTracking(...)` but also contains annoyances
     * rules to block things like cookie notices.
     */ public static fromPrebuiltFull < T extends typeof FilterEngine > ( this , fetchImpl = fetch , caching ? ) { return this . fromLists ( fetchImpl , fullLists , { } , caching ) ; } public static parse < T extends FilterEngine > ( this , filters , options = { } ) { const config = new Config ( options ) ; return new this ( { ... parseFilters ( filters , config ) , config , } ) ; } public static deserialize < T extends FilterEngine > ( this , serialized ) { const buffer = StaticDataView . fromUint8Array ( serialized , { enableCompression : false , } ) ; const serializedEngineVersion = buffer . getUint16 ( ) ; if ( ENGINE_VERSION !== serializedEngineVersion ) { throw new Error ( `serialized engine version mismatch, expected ${ ENGINE_VERSION } but got ${ serializedEngineVersion }` ) ; } const config = Config . deserialize ( buffer ) ; if ( config . enableCompression ) { buffer . enableCompression ( ) ; } if ( config . integrityCheck ) { const currentPos = buffer . pos ; buffer . pos = serialized . length - 4 ; const checksum = buffer . checksum ( ) ; const expected = buffer . getUint32 ( ) ; if ( checksum !== expected ) { throw new Error ( `serialized engine checksum mismatch, expected ${ expected } but got ${ checksum }` ) ; } buffer . pos = currentPos ; } const engine = new this ( { config } ) ; engine . resources = Resources . deserialize ( buffer ) ; const lists = new Map ( ) ; const numberOfLists = buffer . getUint16 ( ) ; for ( let i = 0 ; i < numberOfLists ; i += 1 ) { lists . set ( buffer . getASCII ( ) , buffer . getASCII ( ) ) ; } engine . lists = lists ; engine . importants = NetworkFilterBucket . deserialize ( buffer , config ) ; engine . redirects = NetworkFilterBucket . deserialize ( buffer , config ) ; engine . filters = NetworkFilterBucket . deserialize ( buffer , config ) ; engine . exceptions = NetworkFilterBucket . deserialize ( buffer , config ) ; engine . csp = NetworkFilterBucket . deserialize ( buffer , config ) ; engine . cosmetics = CosmeticFilterBucket . deserialize ( buffer , config ) ; engine . hideExceptions = NetworkFilterBucket . deserialize ( buffer , config ) ; return engine ; } public lists : Map < string , string > ; public csp : NetworkFilterBucket ; public hideExceptions : NetworkFilterBucket ; public exceptions : NetworkFilterBucket ; public importants : NetworkFilterBucket ; public redirects : NetworkFilterBucket ; public filters : NetworkFilterBucket ; public cosmetics : CosmeticFilterBucket ; public resources : Resources ; public readonly config : Config ; constructor ( { cosmeticFilters = [ ] , networkFilters = [ ] , config = new Config ( ) , lists = new Map ( ) , } = { } ) { super ( ) ; this . config = new Config ( config ) ; this . lists = lists ; this . csp = new NetworkFilterBucket ( { config : this . config } ) ; this . hideExceptions = new NetworkFilterBucket ( { config : this . config } ) ; this . exceptions = new NetworkFilterBucket ( { config : this . config } ) ; this . importants = new NetworkFilterBucket ( { config : this . config } ) ; this . redirects = new NetworkFilterBucket ( { config : this . config } ) ; this . filters = new NetworkFilterBucket ( { config : this . config } ) ; this . cosmetics = new CosmeticFilterBucket ( { config : this . config } ) ; this . resources = new Resources ( ) ; if ( networkFilters . length !== 0 || cosmeticFilters . length !== 0 ) { this . update ( { newCosmeticFilters : cosmeticFilters , newNetworkFilters : networkFilters , } ) ; } } /**
     * Estimate the number of bytes needed to serialize this instance of
     * `FiltersEngine` using the `serialize(...)` method. It is used internally
     * by `serialize(...)` to allocate a buffer of the right size and you should
     * not have to call it yourself most of the time.
     *
     * There are cases where we cannot estimate statically the exact size of the
     * resulting buffer (due to alignement which need to be performed); this
     * method will return a safe estimate which will always be at least equal to
     * the real number of bytes needed, or bigger (usually of a few bytes only:
     * ~20 bytes is to be expected).
     */ public getSerializedSize ( ) { let estimatedSize = sizeOfByte ( ) + this . config . getSerializedSize ( ) + this . resources . getSerializedSize ( ) + this . filters . getSerializedSize ( ) + this . exceptions . getSerializedSize ( ) + this . importants . getSerializedSize ( ) + this . redirects . getSerializedSize ( ) + this . csp . getSerializedSize ( ) + this . cosmetics . getSerializedSize ( ) + this . hideExceptions . getSerializedSize ( ) + 4 ; for ( const [ name , checksum ] of this . lists ) { estimatedSize += sizeOfASCII ( name ) + sizeOfASCII ( checksum ) ; } return estimatedSize ; } /**
     * Creates a binary representation of the full engine. It can be stored
     * on-disk for faster loading of the adblocker. The `deserialize` static
     * method of Engine can be used to restore the engine.
     */ public serialize ( array ? ) { const buffer = StaticDataView . fromUint8Array ( array || new Uint8Array ( this . getSerializedSize ( ) ) , this . config ) ; buffer . pushUint16 ( ENGINE_VERSION ) ; this . config . serialize ( buffer ) ; this . resources . serialize ( buffer ) ; buffer . pushUint16 ( this . lists . size ) ; for ( const [ name , value ] of Array . from ( this . lists . entries ( ) ) . sort ( ) ) { buffer . pushASCII ( name ) ; buffer . pushASCII ( value ) ; } this . importants . serialize ( buffer ) ; this . redirects . serialize ( buffer ) ; this . filters . serialize ( buffer ) ; this . exceptions . serialize ( buffer ) ; this . csp . serialize ( buffer ) ; this . cosmetics . serialize ( buffer ) ; this . hideExceptions . serialize ( buffer ) ; if ( this . config . integrityCheck ) { buffer . pushUint32 ( buffer . checksum ( ) ) ; } return buffer . subarray ( ) ; } /**
     * Update engine with new filters or resources.
     */ public loadedLists ( ) { return Array . from ( this . lists . keys ( ) ) ; } public hasList ( name , checksum ) { return this . lists . has ( name ) && this . lists . get ( name ) === checksum ; } /**
     * Update engine with `resources.txt` content.
     */ public updateResources ( data , checksum ) { if ( this . resources . checksum === checksum ) { return false ; } this . resources = Resources . parse ( data , { checksum } ) ; return true ; } public getFilters ( ) { const cosmeticFilters = [ ] ; const networkFilters = [ ] ; return { cosmeticFilters : cosmeticFilters . concat ( this . cosmetics . getFilters ( ) ) , networkFilters : networkFilters . concat ( this . filters . getFilters ( ) , this . exceptions . getFilters ( ) , this . importants . getFilters ( ) , this . redirects . getFilters ( ) , this . csp . getFilters ( ) , this . hideExceptions . getFilters ( ) ) , } ; } /**
     * Update engine with new filters as well as optionally removed filters.
     */ public update ( { newNetworkFilters = [ ] , newCosmeticFilters = [ ] , removedCosmeticFilters = [ ] , removedNetworkFilters = [ ] , } ) { let updated = false ; if ( this . config . loadCosmeticFilters && ( newCosmeticFilters . length !== 0 || removedCosmeticFilters . length !== 0 ) ) { updated = true ; this . cosmetics . update ( newCosmeticFilters , removedCosmeticFilters . length === 0 ? undefined : new Set ( removedCosmeticFilters ) , this . config ) ; } if ( this . config . loadNetworkFilters && ( newNetworkFilters . length !== 0 || removedNetworkFilters . length !== 0 ) ) { updated = true ; const filters = [ ] ; const csp = [ ] ; const exceptions = [ ] ; const importants = [ ] ; const redirects = [ ] ; const hideExceptions = [ ] ; for ( const filter of newNetworkFilters ) { if ( filter . isCSP ( ) ) { csp . push ( filter ) ; } else if ( filter . isGenericHide ( ) || filter . isSpecificHide ( ) ) { hideExceptions . push ( filter ) ; } else if ( filter . isException ( ) ) { exceptions . push ( filter ) ; } else if ( filter . isImportant ( ) ) { importants . push ( filter ) ; } else if ( filter . isRedirect ( ) ) { redirects . push ( filter ) ; } else { filters . push ( filter ) ; } } const removedNetworkFiltersSet = removedNetworkFilters . length === 0 ? undefined : new Set ( removedNetworkFilters ) ; this . importants . update ( importants , removedNetworkFiltersSet ) ; this . redirects . update ( redirects , removedNetworkFiltersSet ) ; this . filters . update ( filters , removedNetworkFiltersSet ) ; if ( this . config . loadExceptionFilters === true ) { this . exceptions . update ( exceptions , removedNetworkFiltersSet ) ; } if ( this . config . loadCSPFilters === true ) { this . csp . update ( csp , removedNetworkFiltersSet ) ; } this . hideExceptions . update ( hideExceptions , removedNetworkFiltersSet ) ; } return updated ; } public updateFromDiff ( { added , removed } ) { const newCosmeticFilters = [ ] ; const newNetworkFilters = [ ] ; const removedCosmeticFilters = [ ] ; const removedNetworkFilters = [ ] ; if ( removed !== undefined && removed . length !== 0 ) { const { networkFilters , cosmeticFilters } = parseFilters ( removed . join ( '\n' ) , this . config ) ; Array . prototype . push . apply ( removedCosmeticFilters , cosmeticFilters ) ; Array . prototype . push . apply ( removedNetworkFilters , networkFilters ) ; } if ( added !== undefined && added . length !== 0 ) { const { networkFilters , cosmeticFilters } = parseFilters ( added . join ( '\n' ) , this . config ) ; Array . prototype . push . apply ( newCosmeticFilters , cosmeticFilters ) ; Array . prototype . push . apply ( newNetworkFilters , networkFilters ) ; } return this . update ( { newCosmeticFilters , newNetworkFilters , removedCosmeticFilters : removedCosmeticFilters . map ( ( f ) => f . getId ( ) ) , removedNetworkFilters : removedNetworkFilters . map ( ( f ) => f . getId ( ) ) , } ) ; } /**
     * Return a list of HTML filtering rules.
     */ public getHtmlFilters ( { url , hostname , domain , } ) { const htmlSelectors = [ ] ; if ( this . config . enableHtmlFiltering === false || this . config . loadCosmeticFilters === false ) { return htmlSelectors ; } const rules = this . cosmetics . getHtmlRules ( { domain : domain || '' , hostname , } ) ; for ( const rule of rules ) { const extended = rule . getExtendedSelector ( ) ; if ( extended !== undefined ) { htmlSelectors . push ( extended ) ; } } if ( htmlSelectors . length !== 0 ) { this . emit ( 'html-filtered' , htmlSelectors , url ) ; } return htmlSelectors ; } /**
     * Given `hostname` and `domain` of a page (or frame), return the list of
     * styles and scripts to inject in the page.
     */ public getCosmeticsFilters ( { url , hostname , domain , classes , hrefs , ids , getBaseRules = true , getInjectionRules = true , getExtendedRules = true , getRulesFromDOM = true , getRulesFromHostname = true , } ) { if ( this . config . loadCosmeticFilters === false ) { return { active : false , extended : [ ] , scripts : [ ] , styles : '' , } ; } let allowGenericHides = true ; let allowSpecificHides = true ; const exceptions = this . hideExceptions . matchAll ( Request . fromRawDetails ( { domain : domain || '' , hostname , url , sourceDomain : '' , sourceHostname : '' , sourceUrl : '' , } ) ) ; const genericHides = [ ] ; const specificHides = [ ] ; for ( const filter of exceptions ) { if ( filter . isElemHide ( ) ) { allowGenericHides = false ; allowSpecificHides = false ; break ; } if ( filter . isSpecificHide ( ) ) { specificHides . push ( filter ) ; } else if ( filter . isGenericHide ( ) ) { genericHides . push ( filter ) ; } } if ( allowGenericHides === true ) { allowGenericHides = shouldApplyHideException ( genericHides ) === false ; } if ( allowSpecificHides === true ) { allowSpecificHides = shouldApplyHideException ( specificHides ) === false ; } const { injections , stylesheet , extended } = this . cosmetics . getCosmeticsFilters ( { domain : domain || '' , hostname , classes , hrefs , ids , allowGenericHides , allowSpecificHides , getBaseRules , getInjectionRules , getExtendedRules , getRulesFromDOM , getRulesFromHostname , } ) ; const scripts = [ ] ; for ( const injection of injections ) { const script = injection . getScript ( this . resources . js ) ; if ( script !== undefined ) { this . emit ( 'script-injected' , script , url ) ; scripts . push ( script ) ; } } if ( stylesheet . length !== 0 ) { this . emit ( 'style-injected' , stylesheet , url ) ; } return { active : true , extended , scripts , styles : stylesheet , } ; } /**
     * Given a `request`, return all matching network filters found in the engine.
     */ public matchAll ( request ) { const filters = [ ] ; if ( request . isSupported ) { Array . prototype . push . apply ( filters , this . importants . matchAll ( request ) ) ; Array . prototype . push . apply ( filters , this . filters . matchAll ( request ) ) ; Array . prototype . push . apply ( filters , this . exceptions . matchAll ( request ) ) ; Array . prototype . push . apply ( filters , this . csp . matchAll ( request ) ) ; Array . prototype . push . apply ( filters , this . hideExceptions . matchAll ( request ) ) ; Array . prototype . push . apply ( filters , this . redirects . matchAll ( request ) ) ; } return new Set ( filters ) ; } /**
     * Given a "main_frame" request, check if some content security policies
     * should be injected in the page.
     */ public getCSPDirectives ( request ) { if ( ! this . config . loadNetworkFilters ) { return undefined ; } if ( request . isSupported !== true || request . isMainFrame ( ) === false ) { return undefined ; } const matches = this . csp . matchAll ( request ) ; if ( matches . length === 0 ) { return undefined ; } const disabledCsp = new Set ( ) ; const enabledCsp = new Set ( ) ; for ( const filter of matches ) { if ( filter . isException ( ) ) { if ( filter . csp === undefined ) { return undefined ; } disabledCsp . add ( filter . csp ) ; } else { enabledCsp . add ( filter . csp ) ; } } const csps = Array . from ( enabledCsp ) . filter ( ( csp ) => ! disabledCsp . has ( csp ) ) . join ( '; ' ) || undefined ; if ( csps !== undefined ) { this . emit ( 'csp-injected' , csps , request ) ; } return csps ; } /**
     * Decide if a network request (usually from WebRequest API) should be
     * blocked, redirected or allowed.
     */ public match ( request ) { const result = { exception : undefined , filter : undefined , match : false , redirect : undefined , } ; if ( ! this . config . loadNetworkFilters ) { return result ; } if ( request . isSupported ) { result . filter = this . importants . match ( request ) ; let redirectNone ; let redirectRule ; if ( result . filter === undefined ) { const redirects = this . redirects . matchAll ( request ) ; if ( redirects . length !== 0 ) { for ( const filter of redirects ) { if ( filter . getRedirect ( ) === 'none' ) { redirectNone = filter ; } else if ( filter . isRedirectRule ( ) ) { redirectRule = filter ; } else { result . filter = filter ; } } } if ( result . filter === undefined ) { result . filter = this . filters . match ( request ) ; if ( redirectRule !== undefined && result . filter !== undefined ) { result . filter = redirectRule ; } } if ( result . filter !== undefined ) { result . exception = this . exceptions . match ( request ) ; } } if ( result . filter !== undefined && result . exception === undefined && result . filter . isRedirect ( ) ) { if ( redirectNone !== undefined ) { result . exception = redirectNone ; } else { result . redirect = this . resources . getResource ( result . filter . getRedirect ( ) ) ; } } } result . match = result . exception === undefined && result . filter !== undefined ; if ( result . exception !== undefined ) { this . emit ( 'request-whitelisted' , request , result ) ; } else if ( result . redirect !== undefined ) { this . emit ( 'request-redirected' , request , result ) ; } else if ( result . filter !== undefined ) { this . emit ( 'request-blocked' , request , result ) ; } else { this . emit ( 'request-allowed' , request , result ) ; } return result ; } public blockScripts ( ) { this . updateFromDiff ( { added : [ block ( ) . scripts ( ) . redirectTo ( 'javascript' ) . toString ( ) ] , } ) ; return this ; } public blockImages ( ) { this . updateFromDiff ( { added : [ block ( ) . images ( ) . redirectTo ( 'png' ) . toString ( ) ] , } ) ; return this ; } public blockMedias ( ) { this . updateFromDiff ( { added : [ block ( ) . medias ( ) . redirectTo ( 'mp4' ) . toString ( ) ] , } ) ; return this ; } public blockFrames ( ) { this . updateFromDiff ( { added : [ block ( ) . frames ( ) . redirectTo ( 'html' ) . toString ( ) ] , } ) ; return this ; } public blockFonts ( ) { this . updateFromDiff ( { added : [ block ( ) . fonts ( ) . toString ( ) ] , } ) ; return this ; } public blockStyles ( ) { this . updateFromDiff ( { added : [ block ( ) . styles ( ) . toString ( ) ] , } ) ; return this ; } }