import URL from 'url' ; import pAll from 'p-all' ; import { logger } from '../../logger' ; import { ExternalHostError } from '../../types/errors/external-host-error' ; import * as memCache from '../../util/cache/memory' ; import * as packageCache from '../../util/cache/package' ; import * as hostRules from '../../util/host-rules' ; import { Http , HttpOptions } from '../../util/http' ; import { regEx } from '../../util/regex' ; import { ensureTrailingSlash } from '../../util/url' ; import * as composerVersioning from '../../versioning/composer' ; import type { GetReleasesConfig , ReleaseResult } from '../types' ; import type { AllPackages , PackageMeta , PackagistFile , RegistryFile , RegistryMeta , } from './types' ; export const id = 'packagist' ; export const customRegistrySupport = true ; export const defaultRegistryUrls = [ 'https://packagist.org' ] ; export const defaultVersioning = composerVersioning . id ; export const registryStrategy = 'hunt' ; const http = new Http ( id ) ; function getHostOpts ( url ) { let opts = { } ; const { username , password } = hostRules . find ( { hostType : id , url , } ) ; if ( username && password ) { opts = { ... opts , username , password } ; } return opts ; } async function getRegistryMeta ( regUrl ) { const url = URL . resolve ( ensureTrailingSlash ( regUrl ) , 'packages.json' ) ; const opts = getHostOpts ( url ) ; const res = ( await http . getJson < PackageMeta > ( url , opts ) ) . body ; const meta = { providerPackages : { } , } ; meta . packages = res . packages ; if ( res . includes ) { meta . includesFiles = [ ] ; for ( const [ name , val ] of Object . entries ( res . includes ) ) { const file = { key : name . replace ( val . sha256 , '%hash%' ) , sha256 : val . sha256 , } ; meta . includesFiles . push ( file ) ; } } if ( res [ 'providers-url' ] ) { meta . providersUrl = res [ 'providers-url' ] ; } if ( res [ 'providers-lazy-url' ] ) { meta . providersLazyUrl = res [ 'providers-lazy-url' ] ; } if ( res [ 'provider-includes' ] ) { meta . files = [ ] ; for ( const [ key , val ] of Object . entries ( res [ 'provider-includes' ] ) ) { const file = { key , sha256 : val . sha256 , } ; meta . files . push ( file ) ; } } if ( res . providers ) { for ( const [ key , val ] of Object . entries ( res . providers ) ) { meta . providerPackages [ key ] = val . sha256 ; } } return meta ; } async function getPackagistFile ( regUrl , file ) { const { key , sha256 } = file ; const fileName = key . replace ( '%hash%' , sha256 ) ; const opts = getHostOpts ( regUrl ) ; if ( opts . password || opts . headers ?. authorization ) { return ( await http . getJson < PackagistFile > ( regUrl + '/' + fileName , opts ) ) . body ; } const cacheNamespace = 'datasource-packagist-files' ; const cacheKey = regUrl + key ; const cachedResult = await packageCache . get ( cacheNamespace , cacheKey ) ; if ( cachedResult && cachedResult . sha256 === sha256 ) { return cachedResult . res as Promise < PackagistFile > ; } const res = ( await http . getJson < PackagistFile > ( regUrl + '/' + fileName , opts ) ) . body ; const cacheMinutes = 1440 ; await packageCache . set ( cacheNamespace , cacheKey , { res , sha256 } , cacheMinutes ) ; return res ; } function extractDepReleases ( versions ) { const dep = { releases : null } ; if ( ! versions ) { dep . releases = [ ] ; return dep ; } dep . releases = Object . keys ( versions ) . map ( ( version ) => { const release = versions [ version ] ; dep . homepage = release . homepage || dep . homepage ; if ( release . source ?. url ) { dep . sourceUrl = release . source . url ; } return { version : version . replace ( regEx ( /^v/ ) , '' ) , gitRef : version , releaseTimestamp : release . time , } ; } ) ; return dep ; } async function getAllPackages ( regUrl ) { const registryMeta = await getRegistryMeta ( regUrl ) ; const { packages , providersUrl , providersLazyUrl , files , includesFiles , providerPackages , } = registryMeta ; if ( files ) { const queue = files . map ( ( file ) => ( ) : Promise < PackagistFile > => getPackagistFile ( regUrl , file ) ) ; const resolvedFiles = await pAll ( queue , { concurrency : 5 } ) ; for ( const res of resolvedFiles ) { for ( const [ name , val ] of Object . entries ( res . providers ) ) { providerPackages [ name ] = val . sha256 ; } } } const includesPackages = { } ; if ( includesFiles ) { for ( const file of includesFiles ) { const res = await getPackagistFile ( regUrl , file ) ; if ( res . packages ) { for ( const [ key , val ] of Object . entries ( res . packages ) ) { const dep = extractDepReleases ( val ) ; includesPackages [ key ] = dep ; } } } } const allPackages = { packages , providersUrl , providersLazyUrl , providerPackages , includesPackages , } ; return allPackages ; } function getAllCachedPackages ( regUrl ) { const cacheKey = `packagist-${ regUrl }` ; const cachedResult = memCache . get < Promise < AllPackages | null > > ( cacheKey ) ; if ( cachedResult !== undefined ) { return cachedResult ; } const promisedRes = getAllPackages ( regUrl ) ; memCache . set ( cacheKey , promisedRes ) ; return promisedRes ; } async function packagistOrgLookup ( name ) { const cacheNamespace = 'datasource-packagist-org' ; const cachedResult = await packageCache . get < ReleaseResult > ( cacheNamespace , name ) ; if ( cachedResult ) { return cachedResult ; } let dep = null ; const regUrl = 'https://packagist.org' ; const pkgUrl = URL . resolve ( regUrl , `/p/${ name }.json` ) ; const res = ( await http . getJson < any > ( pkgUrl ) ) . body . packages [ name ] ; if ( res ) { dep = extractDepReleases ( res ) ; logger . trace ( { dep } , 'dep' ) ; } const cacheMinutes = 10 ; await packageCache . set ( cacheNamespace , name , dep , cacheMinutes ) ; return dep ; } async function packageLookup ( regUrl , name ) { try { if ( regUrl === 'https://packagist.org' ) { const packagistResult = await packagistOrgLookup ( name ) ; return packagistResult ; } const allPackages = await getAllCachedPackages ( regUrl ) ; const { packages , providersUrl , providersLazyUrl , providerPackages , includesPackages , } = allPackages ; if ( packages ?. [ name ] ) { const dep = extractDepReleases ( packages [ name ] ) ; return dep ; } if ( includesPackages ?. [ name ] ) { return includesPackages [ name ] ; } let pkgUrl ; if ( name in providerPackages ) { pkgUrl = URL . resolve ( regUrl , providersUrl . replace ( '%package%' , name ) . replace ( '%hash%' , providerPackages [ name ] ) ) ; } else if ( providersLazyUrl ) { pkgUrl = URL . resolve ( regUrl , providersLazyUrl . replace ( '%package%' , name ) ) ; } else { return null ; } const opts = getHostOpts ( regUrl ) ; const versions = ( await http . getJson < any > ( pkgUrl , opts ) ) . body . packages [ name ] ; const dep = extractDepReleases ( versions ) ; logger . trace ( { dep } , 'dep' ) ; return dep ; } catch ( err ) { if ( err . host === 'packagist.org' ) { if ( err . code === 'ECONNRESET' || err . code === 'ETIMEDOUT' ) { throw new ExternalHostError ( err ) ; } if ( err . statusCode && err . statusCode >= 500 && err . statusCode < 600 ) { throw new ExternalHostError ( err ) ; } } throw err ; } } export function getReleases ( { lookupName , registryUrl , } ) { logger . trace ( `getReleases(${ lookupName })` ) ; return packageLookup ( registryUrl , lookupName ) ; }