import commander from "commander" ; import fs from "fs" ; import fsExtra from "fs-extra" ; import mkdirp from "mkdirp" ; import path from "path" ; import process from "process" ; import simpleGit from "simple-git/promise" ; import { loadConfigurationFromLocalEnv , readYaml } from "../../config" ; import { build as buildCmd , exit as exitCmd } from "../../lib/commandBuilder" ; import { safeGitUrlForLogging } from "../../lib/gitutils" ; import { deepClone } from "../../lib/util" ; import { logger } from "../../logger" ; import { InfraConfigYaml } from "../../types" ; import decorator from "./generate.decorator.json" ; import { BACKEND_TFVARS , DEFINITION_YAML , getSourceFolderNameFromURL , BEDROCK_TFVARS , bedrockTemplatesPath , } from "./infra_common" ; import { copyTfTemplate } from "./scaffold" ; import { build as buildError , log as logError } from "../../lib/errorBuilder" ; import { errorStatusCode } from "../../lib/errorStatusCode" ; import { exec } from "../../lib/shell" ; interface CommandOptions { project : string | undefined ; output : string | undefined ; } export interface SourceInformation { source : string ; template : string ; version : string ; } export enum DefinitionYAMLExistence { BOTH_EXIST , PARENT_ONLY } const regexSource = /^\s*source\s*=\s*["'](\.\.?\/[^"']*)["']$/gm ; param parentPath param projectPath export const validateDefinition = ( parentPath , projectPath ) : DefinitionYAMLExistence => { mkdirp . sync ( bedrockTemplatesPath ) ; const parentPathExist = fs . existsSync ( path . join ( parentPath , DEFINITION_YAML ) ) ; const projectPathExist = fs . existsSync ( path . join ( projectPath , DEFINITION_YAML ) ) ; if ( parentPathExist ) { if ( projectPathExist ) { logger . info ( `${ DEFINITION_YAML } was found in ${ parentPath } and ${ projectPath }` ) ; return DefinitionYAMLExistence . BOTH_EXIST ; } logger . warn ( `${ DEFINITION_YAML } was found in: ${ parentPath }, but was not found in ${ projectPath }` ) ; return DefinitionYAMLExistence . PARENT_ONLY ; } throw buildError ( errorStatusCode . ENV_SETTING_ERR , { errorKey : "infra-defn-yaml-not-found" , values : [ DEFINITION_YAML , parentPath ] , } ) ; } ; export const getDefinitionYaml = ( dir ) : InfraConfigYaml => { const parentData = readYaml < InfraConfigYaml > ( path . join ( dir , DEFINITION_YAML ) ) ; return loadConfigurationFromLocalEnv ( parentData || { } ) ; } ; param configuration param parentDir param projectDir export const validateTemplateSources = ( configuration , parentDir , projectDir ) : SourceInformation => { const sourceKeys = [ "source" , "template" , "version" ] as Array < keyof SourceInformation > ; const source = { source : "" , template : "" , version : "" , } ; let parentInfraConfig ; let leafInfraConfig ; if ( configuration === DefinitionYAMLExistence . PARENT_ONLY ) { parentInfraConfig = getDefinitionYaml ( parentDir ) ; } else if ( configuration === DefinitionYAMLExistence . BOTH_EXIST ) { parentInfraConfig = getDefinitionYaml ( parentDir ) ; leafInfraConfig = getDefinitionYaml ( projectDir ) ; } sourceKeys . forEach ( ( k ) => { if ( leafInfraConfig && leafInfraConfig [ k ] ) { source [ k ] = leafInfraConfig [ k ] ; } else if ( parentInfraConfig && parentInfraConfig [ k ] ) { source [ k ] = parentInfraConfig [ k ] ; } } ) ; if ( source . source && source . template && source . version ) { const safeLoggingUrl = safeGitUrlForLogging ( source . source ) ; logger . info ( `Checking for locally stored template: ${ source . template } from remote repository: ${ safeLoggingUrl } at version: ${ source . version }` ) ; return source ; } throw buildError ( errorStatusCode . INCORRECT_DEFINITION , { errorKey : "infra-defn-yaml-invalid" , values : [ DEFINITION_YAML , source . template , source . source , source . version ] , } ) ; } ; param projectPath export const createGenerated = ( projectPath ) : void => { mkdirp . sync ( projectPath ) ; logger . info ( `Created generated directory: ${ projectPath }` ) ; } ; export const gitPull = async ( sourcePath , safeLoggingUrl ) : Promise < void > => { try { await exec ( "git" , [ "symbolic-ref" , "HEAD" ] , { cwd : sourcePath } ) ; logger . info ( `${ safeLoggingUrl } already cloned and a git branch is currently checked out. Performing 'git pull'...` ) ; await simpleGit ( sourcePath ) . pull ( ) ; } catch ( err ) { logger . info ( `A git tag is currently checked out. Skipping 'git pull' operation.` ) ; } } ; export const gitCheckout = async ( sourcePath , version ) : Promise < void > => { logger . info ( `Checking out template version: ${ version }` ) ; await simpleGit ( sourcePath ) . checkout ( version ) ; } ; param source param sourcePath export const gitClone = async ( git , source , sourcePath ) : Promise < void > => { await git . clone ( source , `${ sourcePath }` ) ; logger . info ( `Cloning source repo to .bedrock/templates was successful.` ) ; } ; param sourcePath export const checkSrcPath = async ( sourcePath ) : Promise < void > => { if ( ! fs . existsSync ( sourcePath ) ) { throw buildError ( errorStatusCode . GIT_OPS_ERR , { errorKey : "infra-git-source-no-exist" , values : [ sourcePath ] , } ) ; } } ; param sourcePath param source safeLoggingUrl export const checkRemoteGitExist = async ( sourcePath , source , safeLoggingUrl ) : Promise < void > => { await checkSrcPath ( sourcePath ) ; const result = await simpleGit ( sourcePath ) . listRemote ( [ source ] ) ; if ( ! result ) { logger . error ( result ) ; throw buildError ( errorStatusCode . GIT_OPS_ERR , "infra-err-git-clone-failed" ) ; } logger . info ( `Remote source repo: ${ safeLoggingUrl } exists.` ) ; } ; param source param sourcePath param safeLoggingUrl param version export const retryRemoteValidate = async ( source , sourcePath , safeLoggingUrl , version ) : Promise < void > => { fsExtra . removeSync ( sourcePath ) ; createGenerated ( sourcePath ) ; const git = simpleGit ( ) ; await gitClone ( git , source , sourcePath ) ; logger . info ( `Checking out template version: ${ version }` ) ; await gitCheckout ( sourcePath , version ) ; await gitPull ( sourcePath , safeLoggingUrl ) ; logger . info ( `Successfully re-cloned repo` ) ; } ; param sourceConfig export const validateRemoteSource = async ( sourceConfig ) : Promise < void > => { const source = sourceConfig . source ; const version = sourceConfig . version ; const sourceFolder = getSourceFolderNameFromURL ( source ) ; const sourcePath = path . join ( bedrockTemplatesPath , sourceFolder ) ; const safeLoggingUrl = safeGitUrlForLogging ( source ) ; logger . info ( `Converted to: ${ sourceFolder }` ) ; logger . info ( `Checking if source: ${ sourcePath } is stored locally.` ) ; if ( ! fs . existsSync ( sourcePath ) ) { logger . warn ( `Provided source in template directory was not found, attempting to clone the template source repo locally.` ) ; createGenerated ( sourcePath ) ; } else { logger . info ( `Source template folder found. Validating existence of repository.` ) ; } await checkRemoteGitExist ( sourcePath , source , safeLoggingUrl ) ; logger . info ( `Checking if source repo: ${ safeLoggingUrl } has been already cloned to: ${ sourcePath }.` ) ; try { if ( fs . existsSync ( path . join ( sourcePath , ".git" ) ) ) { logger . info ( `${ source } already cloned. Proceeding with 'git checkout'.` ) ; } else { const git = simpleGit ( ) ; await gitClone ( git , source , sourcePath ) ; } await gitCheckout ( sourcePath , version ) ; await gitPull ( sourcePath , safeLoggingUrl ) ; } catch ( err ) { if ( err instanceof Error ) { let retry = false ; logger . warn ( `FAILURE DETECTED: Checking the error` ) ; try { if ( err . message . includes ( "refusing to merge unrelated histories" ) ) { logger . info ( `Detected a refusal to merge unrelated histories, attempting to reset the cached folder to the remote: ${ sourcePath }` ) ; retry = true ; } else if ( err . message . includes ( "Authentication failed" ) ) { logger . info ( `Detected an authentication failure with existing cache, attempting to reset the cached folder to the remote: ${ sourcePath }` ) ; retry = true ; } if ( retry ) { await retryRemoteValidate ( source , sourcePath , safeLoggingUrl , version ) ; } else { throw buildError ( errorStatusCode . GIT_OPS_ERR , "infra-err-validating-remote-git-after-retry" , err ) ; } } catch ( retryError ) { throw buildError ( errorStatusCode . GIT_OPS_ERR , "infra-err-retry-validating-remote-git" , err ) ; } } else { throw buildError ( errorStatusCode . GIT_OPS_ERR , "infra-err-validating-remote-git" , err ) ; } } } ; export const getParentGeneratedFolder = ( parentPath , outputPath ) : string => { if ( outputPath !== "" ) { const folderName = parentPath . replace ( /^.*[\\/]/ , "" ) ; return path . join ( outputPath , folderName + "-generated" ) ; } return parentPath + "-generated" ; } ; param definition export const generateTfvars = ( definition ) : string [ ] => { if ( ! definition ) { return [ ] ; } return Object . keys ( definition ) . map ( ( k ) => `${ k } = "${ definition [ k ] }"` ) ; } ; param generatedPath param tfvarsFilename export const checkTfvars = ( generatedPath , tfvarsFilename ) : void => { if ( fs . existsSync ( path . join ( generatedPath , tfvarsFilename ) ) ) { fs . unlinkSync ( path . join ( generatedPath , tfvarsFilename ) ) ; } } ; param bedrockTfVars param generatedPath param tfvarsFileName export const writeTfvarsFile = ( bedrockTfVars , generatedPath , tfvarsFilename ) : void => { bedrockTfVars . forEach ( ( tfvar ) => { fs . appendFileSync ( path . join ( generatedPath , tfvarsFilename ) , tfvar + "\n" ) ; } ) ; } ; export const generateConfigWithParentEqProjectPath = async ( parentDirectory , templatePath , parentInfraConfig ) : Promise < void > => { createGenerated ( parentDirectory ) ; if ( parentInfraConfig . variables ) { const bedrockTfvarsObject = generateTfvars ( parentInfraConfig . variables ) ; checkTfvars ( parentDirectory , BEDROCK_TFVARS ) ; writeTfvarsFile ( bedrockTfvarsObject , parentDirectory , BEDROCK_TFVARS ) ; await copyTfTemplate ( templatePath , parentDirectory , true ) ; } else { logger . warn ( `Variables are not defined in the definition.yaml` ) ; } if ( parentInfraConfig . backend ) { const backendTfvarsObject = generateTfvars ( parentInfraConfig . backend ) ; checkTfvars ( parentDirectory , BACKEND_TFVARS ) ; writeTfvarsFile ( backendTfvarsObject , parentDirectory , BACKEND_TFVARS ) ; } else { logger . warn ( `A remote backend configuration is not defined in the definition.yaml` ) ; } } ; param parentObject param leafObject export const dirIteration = ( parentObject , leafObject ) : { [ key ] : string ; } => { if ( ! parentObject ) { return ! leafObject ? { } : deepClone ( leafObject ) ; } if ( ! leafObject ) { return parentObject ; } Object . keys ( leafObject ) . forEach ( ( k ) => { if ( leafObject [ k ] ) { parentObject [ k ] = leafObject [ k ] ; } } ) ; return parentObject ; } ; const combineVariable = ( parentVars , leafVars , childDirectory , fileName ) : void => { const merged = dirIteration ( parentVars , leafVars ) ; const combined = generateTfvars ( merged ) ; checkTfvars ( childDirectory , fileName ) ; writeTfvarsFile ( combined , childDirectory , fileName ) ; } ; param infraConfig param parentDirectory param childDirectory param templatePath export const singleDefinitionGeneration = async ( infraConfig , parentDirectory , childDirectory , templatePath ) : Promise < void > => { createGenerated ( parentDirectory ) ; createGenerated ( childDirectory ) ; const bedrockTfvarsObject = generateTfvars ( infraConfig . variables ) ; checkTfvars ( childDirectory , BEDROCK_TFVARS ) ; writeTfvarsFile ( bedrockTfvarsObject , childDirectory , BEDROCK_TFVARS ) ; if ( infraConfig . backend ) { const backendTfvarsObject = generateTfvars ( infraConfig . backend ) ; checkTfvars ( childDirectory , BACKEND_TFVARS ) ; writeTfvarsFile ( backendTfvarsObject , childDirectory , BACKEND_TFVARS ) ; } await copyTfTemplate ( templatePath , childDirectory , true ) ; } ; param tfFile export const checkModuleSource = ( tfData ) : boolean => { const matches = tfData . match ( regexSource ) ; return matches !== null ; } ; export const revparse = async ( sPath ) : Promise < string > => { return await simpleGit ( sPath ) . revparse ( [ "--show-prefix" ] ) ; } ; param sourceConfig export const moduleSourceModify = async ( fileSource , tfData ) : Promise < string > => { try { let result = "" ; const sourceFolder = getSourceFolderNameFromURL ( fileSource . source ) ; const sourcePath = path . join ( bedrockTemplatesPath , sourceFolder ) ; for ( let line of tfData . split ( /\r?\n/ ) ) { if ( line . match ( regexSource ) !== null ) { const splitLine = line . split ( "=" ) ; const cleanPath = splitLine [ 1 ] . trim ( ) ; const moduleSource = new RegExp ( cleanPath . replace ( /['"]+/g , "" ) , "g" ) ; const repoModulePath = await revparse ( path . join ( sourcePath , fileSource . template , cleanPath . replace ( /["']/g , "" ) ) ) ; const gitSource = "git::" + fileSource . source . concat ( "//" , repoModulePath , "?ref=" , fileSource . version ) ; line = line . replace ( moduleSource , gitSource ) ; } result += line + "\n" ; } return result ; } catch ( err ) { throw buildError ( errorStatusCode . EXE_FLOW_ERR , "infra-module-source-modify-err" , err ) ; } } ; param sourceConfig export const inspectGeneratedSources = async ( childDirectory , sourceConfig ) : Promise < void > => { try { const files = fsExtra . readdirSync ( childDirectory , "utf-8" ) ; for ( const file of files ) { if ( path . extname ( file ) === ".tf" ) { const tfData = fsExtra . readFileSync ( path . join ( childDirectory , file ) , "utf8" ) ; const containsLocalSource = checkModuleSource ( tfData ) ; if ( containsLocalSource ) { logger . info ( `Local relative paths for module source values detected in terraform file: ${ file }` ) ; const mungeData = await moduleSourceModify ( sourceConfig , tfData ) ; logger . info ( `Terraform File: ${ file } local module source values successfully converted to git source paths` ) ; fsExtra . writeFileSync ( path . join ( childDirectory , file ) , mungeData , "utf8" ) ; } } } } catch ( err ) { throw buildError ( errorStatusCode . EXE_FLOW_ERR , "infra-inspect-generated-sources-err" , err ) ; } } ; param parentPath param projectPath param definitionConfig param sourceConfig param outputPath export const generateConfig = async ( parentPath , projectPath , definitionConfig , sourceConfig , outputPath ) : Promise < void > => { const parentDirectory = getParentGeneratedFolder ( parentPath , outputPath ) ; const sourceFolder = getSourceFolderNameFromURL ( sourceConfig . source ) ; const templatePath = path . join ( bedrockTemplatesPath , sourceFolder , sourceConfig . template ) ; const childDirectory = projectPath === parentPath ? parentPath : path . join ( parentDirectory , projectPath ) ; if ( definitionConfig === DefinitionYAMLExistence . BOTH_EXIST ) { const parentInfraConfig = getDefinitionYaml ( parentPath ) ; const leafInfraConfig = getDefinitionYaml ( projectPath ) ; if ( projectPath === parentPath ) { await generateConfigWithParentEqProjectPath ( parentDirectory , templatePath , parentInfraConfig ) ; } else { createGenerated ( parentDirectory ) ; createGenerated ( childDirectory ) ; } combineVariable ( parentInfraConfig . variables ! , leafInfraConfig . variables ! , childDirectory , BEDROCK_TFVARS ) ; if ( parentInfraConfig . backend && leafInfraConfig . backend ) { combineVariable ( parentInfraConfig . backend , leafInfraConfig . backend , childDirectory , BACKEND_TFVARS ) ; } await copyTfTemplate ( templatePath , childDirectory , true ) ; } else if ( definitionConfig === DefinitionYAMLExistence . PARENT_ONLY ) { const parentInfraConfig = getDefinitionYaml ( parentPath ) ; await singleDefinitionGeneration ( parentInfraConfig , parentDirectory , childDirectory , templatePath ) ; } await inspectGeneratedSources ( childDirectory , sourceConfig ) ; } ; export const execute = async ( opts , exitFn ) : Promise < void > => { const parentPath = process . cwd ( ) ; const projectPath = opts . project || parentPath ; const outputPath = opts . output || "" ; try { const definitionConfig = validateDefinition ( parentPath , projectPath ) ; const sourceConfig = validateTemplateSources ( definitionConfig , parentPath , projectPath ) ; await validateRemoteSource ( sourceConfig ) ; await generateConfig ( parentPath , projectPath , definitionConfig , sourceConfig , outputPath ) ; await exitFn ( 0 ) ; } catch ( err ) { logError ( buildError ( errorStatusCode . CMD_EXE_ERR , "infra-generate-cmd-failed" , err ) ) ; await exitFn ( 1 ) ; } } ; param command export const commandDecorator = ( command ) : void => { buildCmd ( command , decorator ) . action ( async ( opts ) => { await execute ( opts , async ( status ) => { await exitCmd ( logger , process . exit , status ) ; } ) ; } ) ; } ;