import * as debugLib from 'debug' ; import * as pathLib from 'path' ; const sortBy = require ( 'lodash.sortby' ) ; const groupBy = require ( 'lodash.groupby' ) ; import { EntityToFix , FixChangesSummary , FixOptions , Issue , RemediationChanges , Workspace , } from '../../../../types' ; import { FixedCache , PluginFixResponse } from '../../../types' ; import { updateDependencies } from './update-dependencies' ; import { NoFixesCouldBeAppliedError } from '../../../../lib/errors/no-fixes-applied' ; import { extractProvenance } from './extract-version-provenance' ; import { ParsedRequirements , parseRequirementsFile , } from './update-dependencies/requirements-file-parser' ; import { standardizePackageName } from '../../standardize-package-name' ; import { containsRequireDirective } from './contains-require-directive' ; import { validateRequiredData } from '../validate-required-data' ; import { formatDisplayName } from '../../../../lib/output-formatters/format-display-name' ; const debug = debugLib ( 'snyk-fix:python:requirements.txt' ) ; export async function pipRequirementsTxt ( fixable , options ) { debug ( `Preparing to fix ${ fixable . length } Python requirements.txt projects` ) ; const handlerResult = { succeeded : [ ] , failed : [ ] , skipped : [ ] , } ; const ordered = sortByDirectory ( fixable ) ; let fixedFilesCache = { } ; for ( const dir of Object . keys ( ordered ) ) { debug ( `Fixing entities in directory ${ dir }` ) ; const entitiesPerDirectory = ordered [ dir ] . map ( ( e ) => e . entity ) ; const { failed , succeeded , skipped , fixedCache } = await fixAll ( entitiesPerDirectory , options , fixedFilesCache ) ; fixedFilesCache = { ... fixedFilesCache , ... fixedCache , } ; handlerResult . succeeded . push ( ... succeeded ) ; handlerResult . failed . push ( ... failed ) ; handlerResult . skipped . push ( ... skipped ) ; } return handlerResult ; } async function fixAll ( entities , options , fixedCache ) { const handlerResult = { succeeded : [ ] , failed : [ ] , skipped : [ ] , } ; for ( const entity of entities ) { const targetFile = entity . scanResult . identity . targetFile ! ; try { const { dir , base } = pathLib . parse ( targetFile ) ; const filePath = pathLib . normalize ( pathLib . join ( dir , base ) ) ; if ( Object . keys ( fixedCache ) . includes ( pathLib . normalize ( pathLib . join ( dir , base ) ) ) ) { handlerResult . succeeded . push ( { original : entity , changes : [ { success : true , userMessage : `Fixed through ${ formatDisplayName ( entity . workspace . path , { type : entity . scanResult . identity . type , targetFile : fixedCache [ filePath ] . fixedIn , } ) }` , issueIds : getFixedEntityIssues ( fixedCache [ filePath ] . issueIds , entity . testResult . issues ) , } , ] , } ) ; continue ; } const { changes , fixedMeta } = await applyAllFixes ( entity , options ) ; if ( ! changes . length ) { debug ( 'Manifest has not changed!' ) ; throw new NoFixesCouldBeAppliedError ( ) ; } const uniqueIssueIds = new Set < string > ( ) ; for ( const c of changes ) { c . issueIds . map ( ( i ) => uniqueIssueIds . add ( i ) ) ; } Object . keys ( fixedMeta ) . forEach ( ( f ) => { fixedCache [ f ] = { fixedIn : targetFile , issueIds : Array . from ( uniqueIssueIds ) , } ; } ) ; handlerResult . succeeded . push ( { original : entity , changes } ) ; } catch ( e ) { debug ( `Failed to fix ${ targetFile }.\nERROR: ${ e }` ) ; handlerResult . failed . push ( { original : entity , error : e } ) ; } } return { ... handlerResult , fixedCache } ; } export async function fixIndividualRequirementsTxt ( workspace , dir , entryFileName , fileName , remediation , parsedRequirements , options , directUpgradesOnly ) { const entryFilePath = pathLib . normalize ( pathLib . join ( dir , entryFileName ) ) ; const fullFilePath = pathLib . normalize ( pathLib . join ( dir , fileName ) ) ; const { updatedManifest , changes } = updateDependencies ( parsedRequirements , remediation . pin , directUpgradesOnly , entryFilePath !== fullFilePath ? formatDisplayName ( workspace . path , { type : 'pip' , targetFile : fullFilePath , } ) : undefined ) ; if ( ! changes . length ) { return { changes } ; } if ( ! options . dryRun ) { debug ( 'Writing changes to file' ) ; await workspace . writeFile ( pathLib . join ( dir , fileName ) , updatedManifest ) ; } else { debug ( 'Skipping writing changes to file in --dry-run mode' ) ; } return { changes } ; } export async function applyAllFixes ( entity , options ) { const { remediation , targetFile : entryFileName , workspace , } = validateRequiredData ( entity ) ; const fixedMeta = { } ; const { dir , base } = pathLib . parse ( entryFileName ) ; const provenance = await extractProvenance ( workspace , dir , dir , base ) ; const upgradeChanges = [ ] ; for ( const fileName of Object . keys ( provenance ) ) { const skipApplyingPins = true ; const { changes } = await fixIndividualRequirementsTxt ( workspace , dir , base , fileName , remediation , provenance [ fileName ] , options , skipApplyingPins ) ; upgradeChanges . push ( ... changes ) ; fixedMeta [ pathLib . normalize ( pathLib . join ( dir , fileName ) ) ] = upgradeChanges ; } const toPin = filterOutAppliedUpgrades ( remediation , upgradeChanges ) ; const directUpgradesOnly = false ; const fileForPinning = await selectFileForPinning ( entity ) ; const { changes : pinnedChanges } = await fixIndividualRequirementsTxt ( workspace , dir , base , fileForPinning . fileName , toPin , parseRequirementsFile ( fileForPinning . fileContent ) , options , directUpgradesOnly ) ; return { changes : [ ... upgradeChanges , ... pinnedChanges ] , fixedMeta } ; } function filterOutAppliedUpgrades ( remediation , upgradeChanges ) { const pinRemediation = { ... remediation , pin : { } , } ; const pins = remediation . pin ; const normalizedAppliedRemediation = upgradeChanges . map ( ( c ) => { if ( c . success && c . from ) { const [ pkgName , versionAndMore ] = c . from ?. split ( '@' ) ; return `${ standardizePackageName ( pkgName ) }@${ versionAndMore }` ; } return false ; } ) . filter ( Boolean ) ; for ( const pkgAtVersion of Object . keys ( pins ) ) { const [ pkgName , versionAndMore ] = pkgAtVersion . split ( '@' ) ; if ( ! normalizedAppliedRemediation . includes ( `${ standardizePackageName ( pkgName ) }@${ versionAndMore }` ) ) { pinRemediation . pin [ pkgAtVersion ] = pins [ pkgAtVersion ] ; } } return pinRemediation ; } function sortByDirectory ( entities ) { const mapped = entities . map ( ( e ) => ( { entity : e , ... pathLib . parse ( e . scanResult . identity . targetFile ! ) , } ) ) ; const sorted = sortBy ( mapped , 'dir' ) ; return groupBy ( sorted , 'dir' ) ; } export async function selectFileForPinning ( entity ) { const targetFile = entity . scanResult . identity . targetFile ! ; const { dir , base } = pathLib . parse ( targetFile ) ; const { workspace } = entity ; let fileName = base ; let requirementsTxt = await workspace . readFile ( targetFile ) ; const { containsRequire , matches } = await containsRequireDirective ( requirementsTxt ) ; const constraintsMatch = matches . filter ( ( m ) => m . includes ( 'c' ) ) ; if ( containsRequire && constraintsMatch [ 0 ] ) { fileName = constraintsMatch [ 0 ] [ 2 ] ; requirementsTxt = await workspace . readFile ( pathLib . join ( dir , fileName ) ) ; } return { fileContent : requirementsTxt , fileName } ; } function getFixedEntityIssues ( fixedIssueIds , issues ) { const fixed = [ ] ; for ( const { issueId } of issues ) { if ( fixedIssueIds . includes ( issueId ) ) { fixed . push ( issueId ) ; } } return fixed ; }