import buildDebug from 'debug' ; import _ from 'lodash' ; import lunr from 'lunr' ; import lunrMutable from 'lunr-mutable-indexes' ; import { PassThrough , Transform , pipeline } from 'stream' ; import { VerdaccioError } from '@verdaccio/core' ; import { errorUtils , searchUtils } from '@verdaccio/core' ; import { logger } from '@verdaccio/logger' ; import { IProxy , ProxyList , ProxySearchParams } from '@verdaccio/proxy' ; import { Version } from '@verdaccio/types' ; import { LocalStorage } from './local-storage' ; import { Storage } from './storage' ; const debug = buildDebug ( 'verdaccio:storage:search' ) ; export interface ISearchResult { ref : string ; score : number ; } export interface IWebSearch { index : lunrMutable . index ; storage : Storage ; query ( query ) : ISearchResult [ ] ; add ( pkg ) : void ; remove ( name ) : void ; reindex ( ) : void ; configureStorage ( storage ) : void ; } export function removeDuplicates ( results ) { const pkgNames = [ ] ; return results . filter ( ( pkg ) => { if ( pkgNames . includes ( pkg ?. package ?. name ) ) { return false ; } pkgNames . push ( pkg ?. package ?. name ) ; return true ; } ) ; } class TransFormResults extends Transform { public constructor ( options ) { super ( options ) ; } param { string } chunk param { string } encoding param { function } done returns { void } @override
      public _transform ( chunk , _encoding , callback ) { if ( _ . isArray ( chunk ) ) { ( chunk as searchUtils . SearchItem [ ] ) . filter ( ( pkgItem ) => { debug ( `streaming remote pkg name ${ pkgItem ?. package ?. name }` ) ; return true ; } ) . forEach ( ( pkgItem ) => { this . push ( pkgItem ) ; } ) ; return callback ( ) ; } else { debug ( `streaming local pkg name ${ chunk ?. package ?. name }` ) ; this . push ( chunk ) ; return callback ( ) ; } } } export class SearchManager { public readonly uplinks : ProxyList ; public readonly localStorage : LocalStorage ; constructor ( uplinks , storage ) { this . uplinks = uplinks ; this . localStorage = storage ; } public get proxyList ( ) { const uplinksList = Object . keys ( this . uplinks ) ; return uplinksList ; } /**
     * Handle search on packages and proxies.
     * Iterate all proxies configured and search in all endpoints in v2 and pipe all responses
     * to a stream, once the proxies request has finished search in local storage for all packages
     * (privated and cached).
     */ public async search ( options ) { const transformResults = new TransFormResults ( { objectMode : true } ) ; const streamPassThrough = new PassThrough ( { objectMode : true } ) ; const upLinkList = this . proxyList ; const searchUplinksStreams = upLinkList . map ( ( uplinkId ) => { const uplink = this . uplinks [ uplinkId ] ; if ( ! uplink ) { logger . fatal ( { uplinkId } , 'uplink @upLinkId not found' ) ; throw new Error ( `uplink ${ uplinkId } not found` ) ; } return this . consumeSearchStream ( uplinkId , uplink , options , streamPassThrough ) ; } ) ; try { debug ( 'search uplinks' ) ; await Promise . all ( [ ... searchUplinksStreams ] ) ; debug ( 'search uplinks done' ) ; } catch ( err ) { logger . error ( { err } , ' error on uplinks search @{err}' ) ; streamPassThrough . emit ( 'error' , err ) ; throw err ; } debug ( 'search local' ) ; await this . localStorage . search ( streamPassThrough , options . query as searchUtils . SearchQuery ) ; const data = [ ] ; const outPutStream = new PassThrough ( { objectMode : true } ) ; pipeline ( streamPassThrough , transformResults , outPutStream , ( err ) => { if ( err ) { throw errorUtils . getInternalError ( err ? err . message : 'unknown error' ) ; } else { debug ( 'Pipeline succeeded.' ) ; } } ) ; outPutStream . on ( 'data' , ( chunk ) => { data . push ( chunk ) ; } ) ; return new Promise ( ( resolve ) => { outPutStream . on ( 'finish' , async ( ) => { const checkAccessPromises = removeDuplicates ( data ) ; debug ( 'stream finish event %s' , checkAccessPromises . length ) ; return resolve ( checkAccessPromises ) ; } ) ; debug ( 'search done' ) ; } ) ; } /**
     * Consume the upstream and pipe it to a transformable stream.
     */ private consumeSearchStream ( uplinkId , uplink , options , searchPassThrough ) { return uplink . search ( { ... options } ) . then ( ( bodyStream ) => { bodyStream . pipe ( searchPassThrough , { end : false } ) ; bodyStream . on ( 'error' , ( err ) : void => { logger . error ( { uplinkId , err : err } , 'search error for uplink @{uplinkId}: @{err?.message}' ) ; searchPassThrough . end ( ) ; } ) ; return new Promise ( ( resolve ) => bodyStream . on ( 'end' , resolve ) ) ; } ) ; } } /**
 * Handle the search Indexer.
 */ class Search implements IWebSearch { public readonly index : lunrMutable . index ; public storage : Storage ; /**
     * Constructor.
     */ public constructor ( ) { this . index = lunrMutable ( function ( ) : void { this . field ( 'name' , { boost : 10 } ) ; this . field ( 'description' , { boost : 4 } ) ; this . field ( 'author' , { boost : 6 } ) ; this . field ( 'keywords' , { boost : 7 } ) ; this . field ( 'version' ) ; this . field ( 'readme' ) ; } ) ; this . index . builder . pipeline . remove ( lunr . stemmer ) ; } public init ( ) { return Promise . resolve ( ) ; } param { * } q return { Array } public query ( query ) { const localStorage = this . storage . localStorage as LocalStorage ; return query === '*' ? ( localStorage . storagePlugin as any ) . get ( ( items ) : any => { items . map ( function ( pkg ) : any { return { ref : pkg , score : 1 } ; } ) ; } ) : this . index . search ( `*${ query }*` ) ; } param { * } pkg public add ( pkg ) { this . index . add ( { id : pkg . name , name : pkg . name , description : pkg . description , version : `v${ pkg . version }` , keywords : pkg . keywords , author : pkg . _npmUser ? pkg . _npmUser . name : '???' , } ) ; } param { * } name public remove ( name ) { this . index . remove ( { id : name } ) ; } /**
     * Force a re-index.
     */ public reindex ( ) { this . storage . getLocalDatabase ( ( error , packages ) : void => { if ( error ) { throw error ; } let i = packages . length ; while ( i -- ) { this . add ( packages [ i ] ) ; } } ) ; } param { * } storage public configureStorage ( storage ) { this . storage = storage ; this . reindex ( ) ; } } const SearchInstance = new Search ( ) ; export { SearchInstance } ;