import type { SearchClient , SearchIndex } from 'algoliasearch' ; import type { QueueObject } from 'async' ; import { queue } from 'async' ; import chalk from 'chalk' ; import type { StateManager } from './StateManager' ; import * as algolia from './algolia' ; import { config } from './config' ; import * as npm from './npm' ; import type { PrefetchedPkg } from './npm/Prefetcher' ; import { Prefetcher } from './npm/Prefetcher' ; import { isFailure } from './npm/types' ; import { saveDoc } from './saveDocs' ; import { datadog } from './utils/datadog' ; import { log } from './utils/log' ; import * as sentry from './utils/sentry' ; import { wait } from './utils/wait' ; export class Bootstrap { stateManager ; algoliaClient ; mainIndex ; bootstrapIndex ; prefetcher : Prefetcher | undefined ; consumer : QueueObject < PrefetchedPkg > | undefined ; constructor ( stateManager , algoliaClient , mainIndex , bootstrapIndex ) { this . stateManager = stateManager ; this . algoliaClient = algoliaClient ; this . mainIndex = mainIndex ; this . bootstrapIndex = bootstrapIndex ; } /**
     * Bootstrap is the mode that goes from 0 to all the packages in NPM
     * In other word it is reindexing everything from scratch.
     *
     * It is useful if:
     *  - you are starting this project for the first time
     *  - you messed up with your Algolia index
     *  - you lagged too much behind.
     *
     * Watch mode should/can be reliably left running for weeks/months as CouchDB is made for that.
     * BUT for the moment it's mandatory to relaunch it because it's the only way to update: typescript, downloads stats.
     */ async run ( ) { log . info ( '-----' ) ; log . info ( '⛷   Bootstrap: starting' ) ; const state = await this . stateManager . check ( ) ; if ( state . seq && state . seq > 0 && state . bootstrapDone === true ) { await algolia . putDefaultSettings ( this . mainIndex , config ) ; log . info ( '⛷   Bootstrap: done' ) ; log . info ( '-----' ) ; return ; } await this . stateManager . save ( { stage : 'bootstrap' , } ) ; const { seq , nbDocs : totalDocs } = await npm . getInfo ( ) ; if ( ! state . bootstrapLastId ) { log . info ( '⛷   Bootstrap: starting from the first doc' ) ; await this . stateManager . save ( { seq } ) ; await algolia . putDefaultSettings ( this . bootstrapIndex , config ) ; } else { log . info ( '⛷   Bootstrap: starting at doc %s' , state . bootstrapLastId ) ; } log . info ( '-----' ) ; log . info ( chalk . yellowBright `Total packages: ${ totalDocs }` ) ; log . info ( '-----' ) ; const prefetcher = new Prefetcher ( { nextKey : state . bootstrapLastId , } ) ; prefetcher . launch ( ) ; let done = 0 ; const consumer = createPkgConsumer ( this . stateManager , this . bootstrapIndex ) ; consumer . unsaturated ( async ( ) => { const next = await prefetcher . getNext ( ) ; consumer . push ( next ) ; done += 1 ; } ) ; consumer . buffer = 0 ; this . prefetcher = prefetcher ; this . consumer = consumer ; let processing = true ; while ( processing ) { this . logProgress ( done ) ; await wait ( config . prefetchWaitBetweenPage ) ; processing = ! prefetcher . isFinished ; done = 0 ; this . consumer . push ( null as any ) ; } if ( this . consumer . length ( ) > 0 ) { await this . consumer . drain ( ) ; } this . consumer . kill ( ) ; this . onDone ( ) ; } async onDone ( ) { await this . stateManager . save ( { bootstrapDone : true , bootstrapLastDone : Date . now ( ) , } ) ; await this . moveToProduction ( ) ; log . info ( '-----' ) ; log . info ( '⛷   Bootstrap: done' ) ; log . info ( '-----' ) ; } /**
     * Move algolia index to prod.
     */ async moveToProduction ( ) { log . info ( '🚚  starting move to production' ) ; const currentState = await this . stateManager . get ( ) ; await this . algoliaClient . copyIndex ( config . bootstrapIndexName , config . indexName ) . wait ( ) ; await this . stateManager . save ( currentState ) ; } /**
     * Log approximate progress.
     */ async logProgress ( nbDocs ) { const { nbDocs : totalDocs } = await npm . getInfo ( ) ; const offset = this . prefetcher ! . offset ; log . info ( chalk . dim . italic . white `[progress] %d/%d docs (%s%) (%s prefetched) (%s processing)` , offset + nbDocs , totalDocs , ( ( Math . max ( offset + nbDocs , 1 ) / totalDocs ) * 100 ) . toFixed ( 2 ) , this . prefetcher ! . idleCount , this . consumer ! . running ( ) ) ; } } /**
 * Consume packages.
 */ function createPkgConsumer ( stateManager , index ) { return queue < PrefetchedPkg > ( async ( pkg ) => { if ( ! pkg ) { return ; } log . info ( `Start:` , pkg . id ) ; const start = Date . now ( ) ; try { datadog . increment ( 'packages' ) ; const res = await npm . getDoc ( pkg . id , pkg . value . rev ) ; if ( isFailure ( res ) ) { log . error ( 'Got an error' , res . error ) ; return ; } await saveDoc ( { row : res , index } ) ; const lastId = ( await stateManager . get ( ) ) . bootstrapLastId ; if ( ! lastId || lastId < pkg . id ) { await stateManager . save ( { bootstrapLastId : pkg . id , } ) ; } } catch ( err ) { sentry . report ( err ) ; } finally { log . info ( `Done:` , pkg . id ) ; datadog . timing ( 'loop' , Date . now ( ) - start ) ; } } , config . bootstrapConcurrency ) ; }