import sleep from 'sleep-promise' ; import { execCmd } from './cmd-utils' ; import { confirmAction , envVar , fetchEnv , fetchEnvOrFallback } from './env-utils' ; import { AccountType , generateGenesisFromEnv , generatePrivateKey , generatePublicKey , getAddressFromEnv , privateKeyToAddress , privateKeyToPublicKey , } from './generate_utils' ; import { applyTerraformModule , destroyTerraformModule , getTerraformModuleOutputs , initTerraformModule , planTerraformModule , showTerraformModulePlan , taintTerraformModuleResource , TerraformVars , untaintTerraformModuleResource , } from './terraform' ; import { getGenesisBlockFromGoogleStorage , getProxiesPerValidator , getProxyName , uploadDataToGoogleStorage , uploadGenesisBlockToGoogleStorage , uploadTestnetInfoToGoogleStorage , } from './testnet-utils' ; export interface ProxyIndex { validatorIndex : number ; proxyIndex : number ; } const projectConfig = { 'celo-testnet' : { secretsBucketName : 'celo-testnet-secrets' , stateBucketName : 'celo_tf_state' , } , 'celo-testnet-production' : { secretsBucketName : 'celo-testnet-secrets-prod' , stateBucketName : 'celo_tf_state_prod' , } , } ; const testnetTerraformModule = 'testnet' ; const testnetNetworkTerraformModule = 'testnet-network' ; interface NodeSecrets { ACCOUNT_ADDRESS : string ; BOOTNODE_ENODE_ADDRESS : string ; PRIVATE_KEY : string ; PROXIED_VALIDATOR_ADDRESS ? : string ; PROXY_ENODE_ADDRESSES ? : string ; [ envVar . GETH_ACCOUNT_SECRET ] : string ; [ envVar . MNEMONIC ] : string ; } const testnetEnvVars = { block_time : envVar . BLOCK_TIME , celo_env : envVar . CELOTOOL_CELOENV , gcloud_credentials_path : envVar . GOOGLE_APPLICATION_CREDENTIALS , gcloud_project : envVar . TESTNET_PROJECT_NAME , geth_verbosity : envVar . GETH_VERBOSITY , geth_bootnode_docker_image_repository : envVar . GETH_BOOTNODE_DOCKER_IMAGE_REPOSITORY , geth_bootnode_docker_image_tag : envVar . GETH_BOOTNODE_DOCKER_IMAGE_TAG , geth_metrics : envVar . GETH_ENABLE_METRICS , geth_node_docker_image_repository : envVar . GETH_NODE_DOCKER_IMAGE_REPOSITORY , geth_node_docker_image_tag : envVar . GETH_NODE_DOCKER_IMAGE_TAG , in_memory_discovery_table : envVar . IN_MEMORY_DISCOVERY_TABLE , istanbul_request_timeout_ms : envVar . ISTANBUL_REQUEST_TIMEOUT_MS , network_id : envVar . NETWORK_ID , private_tx_node_count : envVar . PRIVATE_TX_NODES , node_disk_size_gb : envVar . NODE_DISK_SIZE_GB , private_node_disk_size_gb : envVar . PRIVATE_NODE_DISK_SIZE_GB , tx_node_count : envVar . TX_NODES , validator_count : envVar . VALIDATORS , } ; const testnetNetworkEnvVars = { celo_env : envVar . CELOTOOL_CELOENV , gcloud_credentials_path : envVar . GOOGLE_APPLICATION_CREDENTIALS , gcloud_project : envVar . TESTNET_PROJECT_NAME , } ; const testnetResourcesToReset = [ 'module.bootnode.google_compute_instance.bootnode' , 'module.validator.google_compute_instance.validator.*' , 'module.validator.google_compute_disk.validator.*' , 'module.validator.module.proxy.random_id.full_node.*' , 'module.validator.module.proxy.google_compute_instance.full_node.*' , 'module.validator.module.proxy.random_id.full_node_disk.*' , 'module.validator.module.proxy.google_compute_disk.full_node.*' , 'module.tx_node.random_id.full_node.*' , 'module.tx_node.google_compute_instance.full_node.*' , 'module.tx_node.random_id.full_node_disk.*' , 'module.tx_node.google_compute_disk.full_node.*' , 'module.tx_node_private.random_id.full_node.*' , 'module.tx_node_private.google_compute_instance.full_node.*' , 'module.tx_node_private.random_id.full_node_disk.*' , 'module.tx_node_private.google_compute_disk.full_node.*' , 'module.tx_node_lb.random_id.external' , 'module.tx_node_lb.google_compute_instance_group.external' , 'module.tx_node_lb.random_id.internal' , 'module.tx_node_lb.google_compute_instance_group.internal' , ] ; export async function deploy ( celoEnv , generateSecrets , useExistingGenesis , onConfirmFailed ? ) { if ( ! useDefaultNetwork ( ) ) { console . info ( 'First deploying the testnet VPC network' ) ; const networkVars = getTestnetNetworkVars ( celoEnv ) ; await deployModule ( celoEnv , testnetNetworkTerraformModule , networkVars , onConfirmFailed ) ; } const testnetVars = await getTestnetVars ( celoEnv , useExistingGenesis ) ; await deployModule ( celoEnv , testnetTerraformModule , testnetVars , onConfirmFailed , async ( ) => { if ( generateSecrets ) { console . info ( 'Generating and uploading secrets env files to Google Storage...' ) ; await generateAndUploadSecrets ( celoEnv ) ; } } ) ; await uploadTestnetInfoToGoogleStorage ( celoEnv ) ; } export async function deployModule ( celoEnv , terraformModule , vars , onConfirmFailed ? , onConfirmSuccess ? ) { const backendConfigVars = getTerraformBackendConfigVars ( celoEnv , terraformModule ) ; const envType = fetchEnv ( envVar . ENV_TYPE ) ; console . info ( `
    Deploying:
    Terraform Module: ${ terraformModule }
    Celo Env: ${ celoEnv }
    Environment: ${ envType }
  ` ) ; console . info ( 'Initializing...' ) ; await initTerraformModule ( terraformModule , vars , backendConfigVars ) ; console . info ( 'Planning...' ) ; await planTerraformModule ( terraformModule , vars ) ; await confirmAction ( `Are you sure you want to perform the above plan for Celo env ${ celoEnv } in environment ${ envType }?` , onConfirmFailed , onConfirmSuccess ) ; console . info ( 'Applying...' ) ; await applyTerraformModule ( terraformModule ) ; } export async function destroy ( celoEnv ) { const testnetVars = await getTestnetVars ( celoEnv , true ) ; await destroyModule ( celoEnv , testnetTerraformModule , testnetVars ) ; if ( ! useDefaultNetwork ( ) ) { console . info ( 'Destroying the testnet VPC network' ) ; const networkVars = getTestnetNetworkVars ( celoEnv ) ; await destroyModule ( celoEnv , testnetNetworkTerraformModule , networkVars ) ; } } export async function destroyModule ( celoEnv , terraformModule , vars = { } ) { const backendConfigVars = getTerraformBackendConfigVars ( celoEnv , terraformModule ) ; const envType = fetchEnv ( envVar . ENV_TYPE ) ; console . info ( `
    Destroying:
    Terraform Module: ${ terraformModule }
    Celo Env: ${ celoEnv }
    Environment: ${ envType }
  ` ) ; console . info ( 'Initializing...' ) ; await initTerraformModule ( terraformModule , vars , backendConfigVars ) ; console . info ( 'Planning...' ) ; await planTerraformModule ( terraformModule , vars , true ) ; await showTerraformModulePlan ( terraformModule ) ; await confirmAction ( `Are you sure you want to destroy ${ celoEnv } in environment ${ envType }?` ) ; await destroyTerraformModule ( terraformModule , vars ) ; } export async function taintTestnet ( celoEnv ) { console . info ( 'Tainting testnet...' ) ; const vars = await getTestnetVars ( celoEnv , true ) ; const backendConfigVars = getTerraformBackendConfigVars ( celoEnv , testnetTerraformModule ) ; await initTerraformModule ( testnetTerraformModule , vars , backendConfigVars ) ; for ( const resource of testnetResourcesToReset ) { console . info ( `Tainting ${ resource }` ) ; await taintTerraformModuleResource ( testnetTerraformModule , resource ) ; await sleep ( 2000 ) ; } } export async function untaintTestnet ( celoEnv ) { console . info ( 'Untainting testnet...' ) ; const vars = await getTestnetVars ( celoEnv , true ) ; const backendConfigVars = getTerraformBackendConfigVars ( celoEnv , testnetTerraformModule ) ; await initTerraformModule ( testnetTerraformModule , vars , backendConfigVars ) ; for ( const resource of testnetResourcesToReset ) { console . info ( `Untainting ${ resource }` ) ; await untaintTerraformModuleResource ( testnetTerraformModule , resource ) ; await sleep ( 2000 ) ; } } export async function getTestnetOutputs ( celoEnv ) { const vars = await getTestnetVars ( celoEnv , true ) ; const backendConfigVars = getTerraformBackendConfigVars ( celoEnv , testnetTerraformModule ) ; await initTerraformModule ( testnetTerraformModule , vars , backendConfigVars ) ; return getTerraformModuleOutputs ( testnetTerraformModule , vars ) ; } export async function getInternalTxNodeLoadBalancerIP ( celoEnv ) { const fullCmd = getInternalTxNodeLoadBalancerIpCommand ( celoEnv ) ; const [ output ] = await execCmd ( fullCmd ) ; return output . trim ( ) ; } export async function getInternalValidatorIPs ( celoEnv ) { const outputs = await getTestnetOutputs ( celoEnv ) ; return outputs . validator_internal_ip_addresses . value ; } export async function getInternalProxyIPs ( celoEnv ) { const outputs = await getTestnetOutputs ( celoEnv ) ; return outputs . proxy_internal_ip_addresses . value ; } export async function getInternalTxNodeIPs ( celoEnv ) { const outputs = await getTestnetOutputs ( celoEnv ) ; return outputs . tx_node_internal_ip_addresses . value ; } export function getTerraformBackendConfigVars ( celoEnv , terraformModule ) { return { bucket : stateBucketName ( ) , prefix : `${ celoEnv }/${ terraformModule }` , } ; } async function getTestnetVars ( celoEnv , useExistingGenesis ) { let genesisContent = '' ; if ( useExistingGenesis ) { genesisContent = await getGenesisBlockFromGoogleStorage ( celoEnv ) ; } else { generateGenesisFromEnv ( ) ; await uploadGenesisBlockToGoogleStorage ( genesisContent , celoEnv ) ; } const genesisBuffer = Buffer . from ( genesisContent ) ; const domainName = fetchEnv ( envVar . CLUSTER_DOMAIN_NAME ) ; return { ... getEnvVarValues ( testnetEnvVars ) , dns_gcloud_project : 'celo-testnet' , dns_zone_name : dnsZoneName ( domainName ) , ethstats_host : `${ celoEnv }-ethstats.${ domainName }.org` , forno_host : `${ celoEnv }-forno.${ domainName }.org` , gcloud_secrets_bucket : secretsBucketName ( ) , gcloud_secrets_base_path : secretsBasePath ( celoEnv ) , gcloud_vm_service_account_email : `terraform-testnet@${ fetchEnv ( envVar . TESTNET_PROJECT_NAME ) }.iam.gserviceaccount.com` , genesis_content_base64 : genesisBuffer . toString ( 'base64' ) , letsencrypt_email : 'n@celo.org' , network_name : networkName ( celoEnv ) , proxies_per_validator : JSON . stringify ( getProxiesPerValidator ( ) ) , } ; } function getTestnetNetworkVars ( celoEnv ) { return { ... getEnvVarValues ( testnetNetworkEnvVars ) , network_name : networkName ( celoEnv ) , } ; } function getEnvVarValues ( terraformEnvVars ) { const vars = { } ; for ( const key of Object . keys ( terraformEnvVars ) ) { vars [ key ] = fetchEnv ( terraformEnvVars [ key ] ) ; } return vars ; } export async function generateAndUploadSecrets ( celoEnv ) { const bootnodeSecrets = generateBootnodeSecretEnvVars ( ) ; await uploadSecrets ( celoEnv , bootnodeSecrets , 'bootnode' ) ; const txNodeCount = parseInt ( fetchEnv ( envVar . TX_NODES ) , 10 ) ; for ( let i = 0 ; i < txNodeCount ; i ++ ) { const secrets = generateNodeSecretEnvVars ( AccountType . TX_NODE , i ) ; await uploadSecrets ( celoEnv , secrets , `tx-node-${ i }` ) ; } const privateTxNodeCount = parseInt ( fetchEnv ( envVar . PRIVATE_TX_NODES ) , 10 ) ; for ( let i = 0 ; i < privateTxNodeCount ; i ++ ) { const secrets = generateNodeSecretEnvVars ( AccountType . TX_NODE , i , 1000 + i ) ; await uploadSecrets ( celoEnv , secrets , `tx-node-private-${ i }` ) ; } const validatorCount = parseInt ( fetchEnv ( envVar . VALIDATORS ) , 10 ) ; for ( let i = 0 ; i < validatorCount ; i ++ ) { const secrets = generateNodeSecretEnvVars ( AccountType . VALIDATOR , i ) ; await uploadSecrets ( celoEnv , secrets , `validator-${ i }` ) ; } const proxiesPerValidator = getProxiesPerValidator ( ) ; let validatorIndex = 0 ; for ( const proxyCount of proxiesPerValidator ) { for ( let i = 0 ; i < proxyCount ; i ++ ) { const secrets = generateProxySecretEnvVars ( validatorIndex , i ) ; await uploadSecrets ( celoEnv , secrets , `validator-${ validatorIndex }-proxy-${ i }` ) ; } validatorIndex ++ ; } } function uploadSecrets ( celoEnv , secrets , resourceName ) { const cloudStorageFileName = `${ secretsBasePath ( celoEnv ) }/.env.${ resourceName }` ; return uploadDataToGoogleStorage ( secrets , secretsBucketName ( ) , cloudStorageFileName , false , 'text/plain' ) ; } function generateBootnodeSecretEnvVars ( ) { const mnemonic = fetchEnv ( envVar . MNEMONIC ) ; return formatEnvVars ( { NODE_KEY : generatePrivateKey ( mnemonic , AccountType . BOOTNODE , 0 ) , } ) ; } function generateNodeSecretEnvVars ( accountType , index , keyIndex = index ) { const mnemonic = fetchEnv ( envVar . MNEMONIC ) ; const privateKey = generatePrivateKey ( mnemonic , accountType , keyIndex ) ; const secrets = getNodeSecrets ( privateKey ) ; if ( accountType === AccountType . VALIDATOR ) { const proxiesPerValidator = getProxiesPerValidator ( ) ; if ( index < proxiesPerValidator . length ) { const proxyEnodeAddresses = [ ] ; for ( let proxyIndex = 0 ; proxyIndex < proxiesPerValidator [ index ] ; proxyIndex ++ ) { proxyEnodeAddresses . push ( privateKeyToPublicKey ( generateProxyPrivateKey ( index , proxyIndex ) ) ) ; } secrets . PROXY_ENODE_ADDRESSES = proxyEnodeAddresses . join ( ',' ) ; } } return formatEnvVars ( secrets ) ; } function generateProxySecretEnvVars ( validatorIndex , proxyIndex ) { const privateKey = generateProxyPrivateKey ( validatorIndex , proxyIndex ) ; const secrets = getNodeSecrets ( privateKey ) ; secrets . PROXIED_VALIDATOR_ADDRESS = getAddressFromEnv ( AccountType . VALIDATOR , validatorIndex ) ; return formatEnvVars ( secrets ) ; } function generateProxyPrivateKey ( validatorIndex , proxyIndex ) { const mnemonic = fetchEnv ( envVar . MNEMONIC ) ; const index = validatorIndex * 10000 + proxyIndex ; return generatePrivateKey ( mnemonic , AccountType . PROXY , index ) ; } function getNodeSecrets ( privateKey ) { const mnemonic = fetchEnv ( envVar . MNEMONIC ) ; return { ACCOUNT_ADDRESS : privateKeyToAddress ( privateKey ) , BOOTNODE_ENODE_ADDRESS : generatePublicKey ( mnemonic , AccountType . BOOTNODE , 0 ) , PRIVATE_KEY : privateKey , [ envVar . GETH_ACCOUNT_SECRET ] : fetchEnv ( envVar . GETH_ACCOUNT_SECRET ) , [ envVar . MNEMONIC ] : mnemonic , } ; } function formatEnvVars ( envVars ) { return Object . keys ( envVars ) . map ( ( key ) => `${ key }='${ envVars [ key ] }'` ) . join ( '\n' ) ; } function secretsBasePath ( celoEnv ) { return `vm/${ celoEnv }` ; } function useDefaultNetwork ( ) { return ( fetchEnvOrFallback ( envVar . VM_BASED , 'false' ) !== 'true' || fetchEnv ( envVar . KUBERNETES_CLUSTER_NAME ) === 'celo-networks-dev' ) ; } export function networkName ( celoEnv ) { return useDefaultNetwork ( ) ? 'default' : `${ celoEnv }-network` ; } function secretsBucketName ( ) { const config = configForProject ( ) ; return config . secretsBucketName ; } function stateBucketName ( ) { const config = configForProject ( ) ; return config . stateBucketName ; } function configForProject ( ) { const project = fetchEnv ( envVar . TESTNET_PROJECT_NAME ) ; if ( ! projectConfig . hasOwnProperty ( project ) ) { throw new Error ( `No config for project ${ project }` ) ; } return projectConfig [ project ] ; } function dnsZoneName ( domain ) { return `${ domain }-org` ; } export function getVmSshCommand ( instanceName ) { const project = fetchEnv ( envVar . TESTNET_PROJECT_NAME ) ; const zone = fetchEnv ( envVar . KUBERNETES_CLUSTER_ZONE ) ; return `gcloud beta compute --project '${ project }' ssh --zone '${ zone }' ${ instanceName } --tunnel-through-iap` ; } export function getInternalTxNodeLoadBalancerIpCommand ( celoEnv ) { const project = fetchEnv ( envVar . TESTNET_PROJECT_NAME ) ; return `gcloud compute forwarding-rules list --project '${ project }' --filter="name~'${ celoEnv }-tx-node-lb-internal-fwd-rule'" --format='get(IPAddress)'` ; } export async function getNodeVmName ( celoEnv , nodeType , index ? ) { const nodeTypesWithRandomSuffixes = [ 'tx-node' , 'tx-node-private' , 'proxy' ] ; const nodeTypesWithNoIndex = [ 'bootnode' ] ; let instanceName ; if ( nodeTypesWithRandomSuffixes . includes ( nodeType ) ) { instanceName = await getNodeVmNameWithRandomSuffix ( celoEnv , nodeType , index || 0 ) ; } else { instanceName = `${ celoEnv }-${ nodeType }` ; if ( ! nodeTypesWithNoIndex . includes ( nodeType ) && index !== undefined ) { instanceName += `-${ index }` ; } } return instanceName ; } async function getNodeVmNameWithRandomSuffix ( celoEnv , nodeType , index ) { const project = fetchEnv ( envVar . TESTNET_PROJECT_NAME ) ; const baseName = typeof index === 'number' ? `${ celoEnv }-${ nodeType }-${ index }` : getProxyName ( celoEnv , index . validatorIndex , index . proxyIndex ) ; const [ nodeName ] = await execCmd ( `gcloud compute instances list --project '${ project }' --filter="NAME ~ ${ baseName }-.*" --format get\\(NAME\\)` ) ; return nodeName . trim ( ) ; } export function indexCoercer ( value ) { if ( ! value ) { return value ; } const splitValues = value . split ( ':' ) . filter ( ( v ) => v ) ; if ( splitValues . length === 1 ) { return parseInt ( value , 10 ) ; } else if ( splitValues . length === 2 ) { const parsedValues = splitValues . map ( ( v ) => parseInt ( v , 10 ) ) ; const proxyIndex = { validatorIndex : parsedValues [ 0 ] , proxyIndex : parsedValues [ 1 ] , } ; return proxyIndex ; } else { throw new Error ( 'Incorrect index' ) ; } }