import is from '@sindresorhus/is' ; import { dequal } from 'dequal' ; import { HOST_DISABLED } from '../constants/error-messages' ; import { logger } from '../logger' ; import { ExternalHostError } from '../types/errors/external-host-error' ; import * as memCache from '../util/cache/memory' ; import * as packageCache from '../util/cache/package' ; import { clone } from '../util/clone' ; import { regEx } from '../util/regex' ; import { trimTrailingSlash } from '../util/url' ; import * as allVersioning from '../versioning' ; import datasources from './api' ; import { addMetaData } from './metadata' ; import type { DatasourceApi , DigestConfig , GetPkgReleasesConfig , GetReleasesConfig , ReleaseResult , } from './types' ; export * from './types' ; export { isGetPkgReleasesConfig } from './common' ; export const getDatasources = ( ) : Map < string , DatasourceApi > => datasources ; export const getDatasourceList = ( ) : string [ ] => Array . from ( datasources . keys ( ) ) ; const cacheNamespace = 'datasource-releases' ; function getDatasourceFor ( datasource ) { return datasources . get ( datasource ) ; } type GetReleasesInternalConfig = GetReleasesConfig & GetPkgReleasesConfig ; function logError ( datasource , lookupName , err ) { const { statusCode , code : errCode , url } = err ; if ( statusCode === 404 ) { logger . debug ( { datasource , lookupName , url } , 'Datasource 404' ) ; } else if ( statusCode === 401 || statusCode === 403 ) { logger . debug ( { datasource , lookupName , url } , 'Datasource unauthorized' ) ; } else if ( errCode ) { logger . debug ( { datasource , lookupName , url , errCode } , 'Datasource connection error' ) ; } else { logger . debug ( { datasource , lookupName , err } , 'Datasource unknown error' ) ; } } async function getRegistryReleases ( datasource , config , registryUrl ) { const cacheKey = `${ datasource . id } ${ registryUrl } ${ config . lookupName }` ; if ( datasource . caching ) { const cachedResult = await packageCache . get < ReleaseResult > ( cacheNamespace , cacheKey ) ; if ( cachedResult ) { logger . trace ( { cacheKey } , 'Returning cached datasource response' ) ; return cachedResult ; } } const res = await datasource . getReleases ( { ... config , registryUrl } ) ; if ( res ?. releases . length ) { res . registryUrl ??= registryUrl ; } if ( datasource . caching && res && ! res . isPrivate ) { logger . trace ( { cacheKey } , 'Caching datasource response' ) ; const cacheMinutes = 15 ; await packageCache . set ( cacheNamespace , cacheKey , res , cacheMinutes ) ; } return res ; } function firstRegistry ( config , datasource , registryUrls ) { if ( registryUrls . length > 1 ) { logger . warn ( { datasource : datasource . id , depName : config . depName , registryUrls } , 'Excess registryUrls found for datasource lookup - using first configured only' ) ; } const registryUrl = registryUrls [ 0 ] ; return getRegistryReleases ( datasource , config , registryUrl ) ; } async function huntRegistries ( config , datasource , registryUrls ) { let res ; let caughtError ; for ( const registryUrl of registryUrls ) { try { res = await getRegistryReleases ( datasource , config , registryUrl ) ; if ( res ) { break ; } } catch ( err ) { if ( err instanceof ExternalHostError ) { throw err ; } caughtError = err ; logger . trace ( { err } , 'datasource hunt failure' ) ; } } if ( res ) { return res ; } if ( caughtError ) { throw caughtError ; } return null ; } async function mergeRegistries ( config , datasource , registryUrls ) { let combinedRes ; let caughtError ; for ( const registryUrl of registryUrls ) { try { const res = await getRegistryReleases ( datasource , config , registryUrl ) ; if ( res ) { if ( combinedRes ) { for ( const existingRelease of combinedRes . releases || [ ] ) { existingRelease . registryUrl = combinedRes . registryUrl ; } for ( const additionalRelease of res . releases || [ ] ) { additionalRelease . registryUrl = res . registryUrl ; } combinedRes = { ... res , ... combinedRes } ; delete combinedRes . registryUrl ; combinedRes . releases = [ ... combinedRes . releases , ... res . releases ] ; } else { combinedRes = res ; } } } catch ( err ) { if ( err instanceof ExternalHostError ) { throw err ; } caughtError = err ; logger . trace ( { err } , 'datasource merge failure' ) ; } } if ( combinedRes ?. releases ?. length ) { const seenVersions = new Set < string > ( ) ; combinedRes . releases = combinedRes . releases . filter ( ( release ) => { if ( seenVersions . has ( release . version ) ) { return false ; } seenVersions . add ( release . version ) ; return true ; } ) ; } if ( combinedRes ) { return combinedRes ; } if ( caughtError ) { throw caughtError ; } return null ; } function resolveRegistryUrls ( datasource , extractedUrls ) { const { defaultRegistryUrls = [ ] } = datasource ; if ( ! datasource . customRegistrySupport ) { if ( is . nonEmptyArray ( extractedUrls ) ) { logger . warn ( { datasource : datasource . id , registryUrls : extractedUrls } , 'Custom registries are not allowed for this datasource and will be ignored' ) ; } return defaultRegistryUrls ; } const customUrls = extractedUrls ?. filter ( Boolean ) ; let registryUrls ; if ( is . nonEmptyArray ( customUrls ) ) { registryUrls = [ ... customUrls ] ; } else { registryUrls = [ ... defaultRegistryUrls ] ; } return registryUrls . filter ( Boolean ) . map ( trimTrailingSlash ) ; } export function getDefaultVersioning ( datasourceName ) { const datasource = getDatasourceFor ( datasourceName ) ; if ( ! datasource ) { logger . warn ( { datasourceName } , 'Missing datasource!' ) ; } return datasource ?. defaultVersioning || 'semver' ; } function applyReplacements ( config ) { if ( config . replacementName && config . replacementVersion ) { return { replacementName : config . replacementName , replacementVersion : config . replacementVersion , } ; } return undefined ; } async function fetchReleases ( config ) { const { datasource : datasourceName } = config ; if ( ! datasourceName || getDatasourceFor ( datasourceName ) === undefined ) { logger . warn ( 'Unknown datasource: ' + datasourceName ) ; return null ; } const datasource = getDatasourceFor ( datasourceName ) ; const registryUrls = resolveRegistryUrls ( datasource , config . registryUrls ) ; let dep = null ; const registryStrategy = datasource . registryStrategy || 'hunt' ; try { if ( is . nonEmptyArray ( registryUrls ) ) { if ( registryStrategy === 'first' ) { dep = await firstRegistry ( config , datasource , registryUrls ) ; } else if ( registryStrategy === 'hunt' ) { dep = await huntRegistries ( config , datasource , registryUrls ) ; } else if ( registryStrategy === 'merge' ) { dep = await mergeRegistries ( config , datasource , registryUrls ) ; } } else { dep = await datasource . getReleases ( config ) ; } } catch ( err ) { if ( err . message === HOST_DISABLED || err . err ?. message === HOST_DISABLED ) { return null ; } if ( err instanceof ExternalHostError ) { throw err ; } logError ( datasource . id , config . lookupName , err ) ; } if ( ! dep || dequal ( dep , { releases : [ ] } ) ) { return null ; } addMetaData ( dep , datasourceName , config . lookupName ) ; dep = { ... dep , ... applyReplacements ( config ) } ; return dep ; } function getRawReleases ( config ) { const { datasource , lookupName , registryUrls } = config ; const cacheKey = `${ cacheNamespace }${ datasource }${ lookupName }${ String ( registryUrls ) }` ; const cachedResult = memCache . get < Promise < ReleaseResult | null > > ( cacheKey ) ; if ( cachedResult !== undefined ) { return cachedResult ; } const promisedRes = fetchReleases ( config ) ; memCache . set ( cacheKey , promisedRes ) ; return promisedRes ; } export async function getPkgReleases ( config ) { if ( ! config . datasource ) { logger . warn ( 'No datasource found' ) ; return null ; } const lookupName = config . lookupName || config . depName ; if ( ! lookupName ) { logger . error ( { config } , 'Datasource getReleases without lookupName' ) ; return null ; } let res ; try { res = clone ( await getRawReleases ( { ... config , lookupName , } ) ) ; } catch ( e ) { if ( e instanceof ExternalHostError ) { e . hostType = config . datasource ; e . lookupName = lookupName ; } throw e ; } if ( ! res ) { return res ; } if ( config . extractVersion ) { const extractVersionRegEx = regEx ( config . extractVersion ) ; res . releases = res . releases . map ( ( release ) => { const version = extractVersionRegEx . exec ( release . version ) ?. groups ?. version ; if ( version ) { return { ... release , version } ; } return null ; } ) . filter ( Boolean ) ; } const versioning = config . versioning || getDefaultVersioning ( config . datasource ) ; const version = allVersioning . get ( versioning ) ; res . releases = res . releases . filter ( ( release ) => version . isVersion ( release . version ) ) . sort ( ( a , b ) => version . sortVersions ( a . version , b . version ) ) ; res . releases = res . releases . filter ( ( filterRelease , filterIndex ) => res . releases . findIndex ( ( findRelease ) => findRelease . version === filterRelease . version ) === filterIndex ) ; for ( const [ constraintName , constraintValue ] of Object . entries ( config . constraints || { } ) ) { if ( version . isVersion ( constraintValue ) ) { res . releases = res . releases . filter ( ( release ) => { if ( ! is . nonEmptyArray ( release . constraints ?. [ constraintName ] ) ) { return true ; } return release . constraints [ constraintName ] . some ( ( releaseConstraint ) => ! releaseConstraint || version . matches ( constraintValue , releaseConstraint ) ) ; } ) ; } } res . releases . forEach ( ( release ) => { delete release . constraints ; } ) ; return res ; } export function supportsDigests ( config ) { return 'getDigest' in getDatasourceFor ( config . datasource ) ; } export function getDigest ( config , value ? ) { const datasource = getDatasourceFor ( config . datasource ) ; const lookupName = config . lookupName || config . depName ; const registryUrls = resolveRegistryUrls ( datasource , config . registryUrls ) ; const digestConfig = { registryUrl : registryUrls [ 0 ] , currentValue : config . currentValue , currentDigest : config . currentDigest , lookupName , } ; return datasource . getDigest ( digestConfig , value ) ; } export function getDefaultConfig ( datasource ) { const loadedDatasource = getDatasourceFor ( datasource ) ; return Promise . resolve < Record < string , unknown > > ( loadedDatasource ?. defaultConfig || Object . create ( { } ) ) ; }