import { concurrentMap } from '@celo/utils/lib/async' ; import compareVersions from 'compare-versions' ; import fs from 'fs' ; import { entries , range } from 'lodash' ; import os from 'os' ; import path from 'path' ; import sleep from 'sleep-promise' ; import { GCPClusterConfig } from 'src/lib/k8s-cluster/gcp' ; import stringHash from 'string-hash' ; import { getKubernetesClusterRegion , switchToClusterFromEnv } from './cluster' ; import { execCmd , execCmdWithExitOnFailure , outputIncludes , spawnCmd , spawnCmdWithExitOnFailure , } from './cmd-utils' ; import { EnvTypes , envVar , fetchEnv , fetchEnvOrFallback , isProduction , monorepoRoot , } from './env-utils' ; import { ensureAuthenticatedGcloudAccount } from './gcloud_utils' ; import { generateGenesisFromEnv } from './generate_utils' ; import { buildGethAll , checkoutGethRepo , getEnodesWithExternalIPAddresses , retrieveBootnodeIPAddress , } from './geth' ; import { BaseClusterConfig , CloudProvider } from './k8s-cluster/base' ; import { getStatefulSetReplicas , scaleResource } from './kubernetes' ; import { installPrometheusIfNotExists } from './prometheus' ; import { getGenesisBlockFromGoogleStorage , getProxiesPerValidator , getProxyName , uploadGenesisBlockToGoogleStorage , } from './testnet-utils' ; import { stringToBoolean } from './utils' ; const CLOUDSQL_SECRET_NAME = 'blockscout-cloudsql-credentials' ; const BACKUP_GCS_SECRET_NAME = 'backup-blockchain-credentials' ; const TIMEOUT_FOR_LOAD_BALANCER_POLL = 1000 * 60 * 25 ; const LOAD_BALANCER_POLL_INTERVAL = 1000 * 10 ; const TESTNET_CHART_DIR = '../helm-charts/testnet' ; export type HelmAction = 'install' | 'upgrade' ; async function validateExistingCloudSQLInstance ( instanceName ) { await ensureAuthenticatedGcloudAccount ( ) ; try { await execCmd ( `gcloud sql instances describe ${ instanceName }` ) ; } catch ( error ) { console . error ( `Cloud SQL DB ${ instanceName } does not exist, bailing` ) ; console . error ( error ) ; process . exit ( 1 ) ; } } async function failIfSecretMissing ( secretName , namespace ) { try { await execCmd ( `kubectl get secret ${ secretName } --namespace ${ namespace }` ) ; } catch ( error ) { console . error ( `Couldn't retrieve service account secret, cluster is likely not setup correctly for deployment` ) ; console . error ( error ) ; process . exit ( 1 ) ; } } async function copySecret ( secretName , srcNamespace , destNamespace ) { console . info ( `Copying secret ${ secretName } from namespace ${ srcNamespace } to ${ destNamespace }` ) ; await execCmdWithExitOnFailure ( `kubectl get secret ${ secretName } --namespace ${ srcNamespace } -o yaml |\
  grep -v creationTimestamp | grep -v resourceVersion | grep -v selfLink | grep -v uid |\
  sed 's/default/${ destNamespace }/' | kubectl apply --namespace=${ destNamespace } -f -` ) ; } export async function createCloudSQLInstance ( celoEnv , instanceName ) { await ensureAuthenticatedGcloudAccount ( ) ; console . info ( 'Creating Cloud SQL database, this might take a minute or two ...' ) ; await failIfSecretMissing ( CLOUDSQL_SECRET_NAME , 'default' ) ; try { await execCmd ( `gcloud sql instances describe ${ instanceName }` ) ; console . warn ( `A Cloud SQL instance named ${ instanceName } already exists, so in all likelihood you cannot deploy initial with ${ instanceName }` ) ; } catch ( error ) { if ( error . message . trim ( ) !== `Command failed: gcloud sql instances describe ${ instanceName }\nERROR: (gcloud.sql.instances.describe) HTTPError 404: The Cloud SQL instance does not exist.` ) { console . error ( error . message . trim ( ) ) ; process . exit ( 1 ) ; } } try { await execCmd ( `gcloud sql instances create ${ instanceName } --zone ${ fetchEnv ( envVar . KUBERNETES_CLUSTER_ZONE ) } --database-version POSTGRES_9_6 --cpu 1 --memory 4G` ) ; } catch ( error ) { console . error ( error . message . trim ( ) ) ; } const envType = fetchEnv ( envVar . ENV_TYPE ) ; if ( envType !== EnvTypes . DEVELOPMENT ) { try { await execCmdWithExitOnFailure ( `gcloud sql instances create ${ instanceName }-replica --master-instance-name=${ instanceName } --zone ${ fetchEnv ( envVar . KUBERNETES_CLUSTER_ZONE ) }` ) ; } catch ( error ) { console . error ( error . message . trim ( ) ) ; } } await execCmdWithExitOnFailure ( `gcloud sql instances patch ${ instanceName } --backup-start-time 17:00` ) ; const blockscoutDBUsername = Math . random ( ) . toString ( 36 ) . slice ( - 8 ) ; const blockscoutDBPassword = Math . random ( ) . toString ( 36 ) . slice ( - 8 ) ; console . info ( 'Creating SQL user' ) ; await execCmdWithExitOnFailure ( `gcloud sql users create ${ blockscoutDBUsername } -i ${ instanceName } --password ${ blockscoutDBPassword }` ) ; console . info ( 'Creating blockscout database' ) ; await execCmdWithExitOnFailure ( `gcloud sql databases create blockscout -i ${ instanceName }` ) ; console . info ( 'Copying blockscout service account secret to namespace' ) ; await copySecret ( CLOUDSQL_SECRET_NAME , 'default' , celoEnv ) ; const [ blockscoutDBConnectionName ] = await execCmdWithExitOnFailure ( `gcloud sql instances describe ${ instanceName } --format="value(connectionName)"` ) ; return [ blockscoutDBUsername , blockscoutDBPassword , blockscoutDBConnectionName . trim ( ) ] ; } async function createAndUploadKubernetesSecretIfNotExists ( secretName , serviceAccountName , celoEnv ) { await switchToClusterFromEnv ( celoEnv ) ; const keyfilePath = `/tmp/${ serviceAccountName }_key.json` ; const secretExists = await outputIncludes ( `kubectl get secrets` , secretName , `secret exists, skipping creation: ${ secretName }` ) ; if ( ! secretExists ) { console . info ( `Creating secret ${ secretName }` ) ; await execCmdWithExitOnFailure ( `gcloud iam service-accounts keys create ${ keyfilePath } --iam-account ${ serviceAccountName }@${ fetchEnv ( envVar . TESTNET_PROJECT_NAME ) }.iam.gserviceaccount.com` ) ; await execCmdWithExitOnFailure ( `kubectl create secret generic ${ secretName } --from-file=credentials.json=${ keyfilePath }` ) ; } } export async function createAndUploadCloudSQLSecretIfNotExists ( serviceAccountName , celoEnv ) { return createAndUploadKubernetesSecretIfNotExists ( CLOUDSQL_SECRET_NAME , serviceAccountName , celoEnv ) ; } export async function createAndUploadBackupSecretIfNotExists ( serviceAccountName , celoEnv ) { return createAndUploadKubernetesSecretIfNotExists ( BACKUP_GCS_SECRET_NAME , serviceAccountName , celoEnv ) ; } export function getServiceAccountName ( prefix ) { return `${ prefix }-${ fetchEnv ( envVar . KUBERNETES_CLUSTER_NAME ) }` . slice ( 0 , 30 ) ; } export async function installGCPSSDStorageClass ( ) { const storageClassExists = await outputIncludes ( `kubectl get storageclass` , `ssd` , `SSD StorageClass exists, skipping install` ) ; if ( ! storageClassExists ) { const gcpSSDHelmChartPath = '../helm-charts/gcp-ssd' ; await execCmdWithExitOnFailure ( `helm upgrade -i gcp-ssd ${ gcpSSDHelmChartPath }` ) ; } } export async function installCertManagerAndNginx ( celoEnv , clusterConfig ? ) { const nginxChartVersion = '3.9.0' ; const nginxChartNamespace = 'default' ; const certManagerExists = ( await outputIncludes ( `helm list -n default` , `cert-manager-cluster-issuers` ) ) || ( await outputIncludes ( `helm list -n cert-manager` , `cert-manager-cluster-issuers` ) ) ; if ( certManagerExists ) { console . info ( 'cert-manager-cluster-issuers exists, skipping install' ) ; } else { await installCertManager ( ) ; } const nginxIngressReleaseExists = await outputIncludes ( `helm list -n default` , `nginx-ingress-release` , `nginx-ingress-release exists, skipping install` ) ; if ( ! nginxIngressReleaseExists ) { const valueFilePath = `/tmp/${ celoEnv }-nginx-testnet-values.yaml` ; await nginxHelmParameters ( valueFilePath , celoEnv , clusterConfig ) ; await helmAddAndUpdateRepos ( ) ; await execCmdWithExitOnFailure ( `helm install \
      -n ${ nginxChartNamespace } \
      --version ${ nginxChartVersion } \
      nginx-ingress-release ingress-nginx/ingress-nginx \
      -f ${ valueFilePath }
    ` ) ; } } async function nginxHelmParameters ( valueFilePath , celoEnv , clusterConfig ? ) { const logFormat = `{"timestamp": "$time_iso8601", "requestID": "$req_id", "proxyUpstreamName":
  "$proxy_upstream_name", "proxyAlternativeUpstreamName": "$proxy_alternative_upstream_name","upstreamStatus":
  "$upstream_status", "upstreamAddr": "$upstream_addr","httpRequest":{"requestMethod":
  "$request_method", "requestUrl": "$host$request_uri", "status": $status,"requestSize":
  "$request_length", "responseSize": "$upstream_response_length", "userAgent":
  "$http_user_agent", "remoteIp": "$remote_addr", "referer": "$http_referer",
  "latency": "$upstream_response_time s", "protocol":"$server_protocol"}}` ; let loadBalancerIP = '' ; if ( clusterConfig == null || clusterConfig ?. cloudProvider === CloudProvider . GCP ) { loadBalancerIP = await getOrCreateNginxStaticIp ( celoEnv , clusterConfig ) ; } const valueFileContent = `
controller:
  autoscaling:
    enabled: "true"
    minReplicas: 1
    maxReplicas: 10
    targetCPUUtilizationPercentage: 80
    targetMemoryUtilizationPercentage: 80
  config:
    log-format-escape-json: "true"
    log-format-upstream: '${ logFormat }'
  metrics:
    enabled: "true"
    service:
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "10254"
  service:
    loadBalancerIP: ${ loadBalancerIP }
  resources:
    requests:
      cpu: 300m
      memory: 600Mi
` ; fs . writeFileSync ( valueFilePath , valueFileContent ) ; } async function getOrCreateNginxStaticIp ( celoEnv , clusterConfig ? ) { const staticIpName = clusterConfig ?. clusterName ? `${ clusterConfig ?. clusterName }-nginx` : `${ celoEnv }-nginx` ; let staticIpAddress ; if ( clusterConfig !== undefined && clusterConfig . hasOwnProperty ( 'zone' ) ) { const zone = ( clusterConfig as GCPClusterConfig ) . zone ; await registerIPAddress ( staticIpName , zone ) ; staticIpAddress = await retrieveIPAddress ( staticIpName , zone ) ; } else { await registerIPAddress ( staticIpName ) ; staticIpAddress = await retrieveIPAddress ( staticIpName ) ; } console . info ( `nginx-ingress static ip --> ${ staticIpName }: ${ staticIpAddress }` ) ; return staticIpAddress ; } export async function helmAddAndUpdateRepos ( ) { await execCmdWithExitOnFailure ( `helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx` ) ; await execCmdWithExitOnFailure ( `helm repo add stable https://charts.helm.sh/stable` ) ; await execCmdWithExitOnFailure ( `helm repo add grafana https://grafana.github.io/helm-charts` ) ; await execCmdWithExitOnFailure ( `helm repo update` ) ; } export async function installCertManager ( ) { const clusterIssuersHelmChartPath = `../helm-charts/cert-manager-cluster-issuers` ; console . info ( 'Create the namespace for cert-manager' ) ; await execCmdWithExitOnFailure ( `kubectl create namespace cert-manager` ) ; console . info ( 'Installing cert-manager CustomResourceDefinitions' ) ; await execCmdWithExitOnFailure ( `kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.2.0/cert-manager.crds.yaml` ) ; console . info ( 'Updating cert-manager-cluster-issuers chart dependencies' ) ; await execCmdWithExitOnFailure ( `helm dependency update ${ clusterIssuersHelmChartPath }` ) ; console . info ( 'Installing cert-manager-cluster-issuers' ) ; await execCmdWithExitOnFailure ( `helm install cert-manager-cluster-issuers ${ clusterIssuersHelmChartPath } -n cert-manager` ) ; } export async function installAndEnableMetricsDeps ( installPrometheus , context ? , clusterConfig ? ) { const kubeStateMetricsReleaseExists = await outputIncludes ( `helm list -n default` , `kube-state-metrics` , `kube-state-metrics exists, skipping install` ) ; if ( ! kubeStateMetricsReleaseExists ) { await execCmdWithExitOnFailure ( `helm install kube-state-metrics stable/kube-state-metrics --set rbac.create=true -n default` ) ; } if ( installPrometheus ) { await installPrometheusIfNotExists ( context , clusterConfig ) ; } } export async function grantRoles ( serviceAccountName , role ) { const projectName = fetchEnv ( envVar . TESTNET_PROJECT_NAME ) ; const serviceAccountFullName = `${ serviceAccountName }@${ projectName }.iam.gserviceaccount.com` ; const commandRolesAlreadyGranted = `gcloud projects get-iam-policy ${ projectName }  \
  --flatten="bindings[].members" \
  --format='table(bindings.role)' \
  --filter="bindings.members:serviceAccount:${ serviceAccountFullName }"` ; const rolesAlreadyGranted = await outputIncludes ( commandRolesAlreadyGranted , role , `Role ${ role } already granted for account ${ serviceAccountFullName }, skipping binding` ) ; if ( ! rolesAlreadyGranted ) { const cmd = `gcloud projects add-iam-policy-binding ${ projectName } ` + `--role=${ role } ` + `--member=serviceAccount:${ serviceAccountFullName }` ; await execCmd ( cmd ) ; } return ; } export async function retrieveCloudSQLConnectionInfo ( celoEnv , instanceName , dbSuffix ) { await validateExistingCloudSQLInstance ( instanceName ) ; const secretName = `${ celoEnv }-blockscout${ dbSuffix }` ; const [ blockscoutDBUsername ] = await execCmdWithExitOnFailure ( `kubectl get secret ${ secretName } -o jsonpath='{.data.DATABASE_USER}' -n ${ celoEnv } | base64 --decode` ) ; const [ blockscoutDBPassword ] = await execCmdWithExitOnFailure ( `kubectl get secret ${ secretName } -o jsonpath='{.data.DATABASE_PASSWORD}' -n ${ celoEnv } | base64 --decode` ) ; const [ blockscoutDBConnectionName ] = await execCmdWithExitOnFailure ( `gcloud sql instances describe ${ instanceName } --format="value(connectionName)"` ) ; return [ blockscoutDBUsername , blockscoutDBPassword , blockscoutDBConnectionName . trim ( ) ] ; } export async function deleteCloudSQLInstance ( instanceName ) { console . info ( `Deleting Cloud SQL instance ${ instanceName }, this might take a minute or two ...` ) ; try { await execCmd ( `gcloud sql instances delete ${ instanceName } --quiet` ) ; } catch { console . info ( `Couldn't delete Cloud SQL instance ${ instanceName } -- skipping` ) ; } return [ '' , '' , '' ] ; } export async function resetCloudSQLInstance ( instanceName ) { await validateExistingCloudSQLInstance ( instanceName ) ; console . info ( 'Deleting blockscout database from instance' ) ; await execCmdWithExitOnFailure ( `gcloud sql databases delete blockscout -i ${ instanceName } --quiet` ) ; console . info ( 'Creating blockscout database' ) ; await execCmdWithExitOnFailure ( `gcloud sql databases create blockscout -i ${ instanceName }` ) ; } export async function registerIPAddress ( name , zone ? ) { console . info ( `Registering IP address ${ name }` ) ; try { await execCmd ( `gcloud compute addresses create ${ name } --region ${ getKubernetesClusterRegion ( zone ) }` ) ; } catch ( error ) { if ( ! error . toString ( ) . includes ( 'already exists' ) ) { console . error ( error ) ; process . exit ( 1 ) ; } } } export async function deleteIPAddress ( name , zone ? ) { console . info ( `Deleting IP address ${ name }` ) ; try { await execCmd ( `gcloud compute addresses delete ${ name } --region ${ getKubernetesClusterRegion ( zone ) } -q` ) ; } catch ( error ) { if ( ! error . toString ( ) . includes ( 'was not found' ) ) { console . error ( error ) ; process . exit ( 1 ) ; } } } export async function retrieveIPAddress ( name , zone ? ) { const regionFlag = zone === 'global' ? '--global' : `--region ${ getKubernetesClusterRegion ( zone ) }` ; const [ address ] = await execCmdWithExitOnFailure ( `gcloud compute addresses describe ${ name } ${ regionFlag } --format="value(address)"` ) ; return address . replace ( /\n*$/ , '' ) ; } export async function retrieveIPAddresses ( prefix , zone ? ) { const [ address ] = await execCmdWithExitOnFailure ( `gcloud compute addresses list --filter="name~'${ prefix }-' AND name!~'${ prefix }-private-' AND region:( ${ getKubernetesClusterRegion ( zone ) } )" --format="value(name)"` ) ; return address . split ( '\n' ) ; } export async function retrieveClusterIPAddress ( resourceType , resourceName , namespace ) { const [ address ] = await execCmdWithExitOnFailure ( `kubectl get ${ resourceType } ${ resourceName } -n ${ namespace } -o jsonpath={.spec.clusterIP}` ) ; return address ; } export async function createStaticIPs ( celoEnv ) { console . info ( `Creating static IPs for ${ celoEnv }` ) ; const numTxNodes = parseInt ( fetchEnv ( envVar . TX_NODES ) , 10 ) ; await concurrentMap ( 5 , range ( numTxNodes ) , ( i ) => registerIPAddress ( `${ celoEnv }-tx-nodes-${ i }` ) ) ; if ( useStaticIPsForGethNodes ( ) ) { await registerIPAddress ( `${ celoEnv }-bootnode` ) ; const validatorCount = parseInt ( fetchEnv ( envVar . VALIDATORS ) , 10 ) ; const proxiesPerValidator = getProxiesPerValidator ( ) ; for ( let i = 0 ; i < validatorCount ; i ++ ) { if ( proxiesPerValidator [ i ] === 0 ) { await registerIPAddress ( `${ celoEnv }-validators-${ i }` ) ; } } let validatorIndex = 0 ; for ( const proxyCount of proxiesPerValidator ) { for ( let i = 0 ; i < proxyCount ; i ++ ) { await registerIPAddress ( getProxyName ( celoEnv , validatorIndex , i ) ) ; } validatorIndex ++ ; } const numPrivateTxNodes = parseInt ( fetchEnv ( envVar . PRIVATE_TX_NODES ) , 10 ) ; await concurrentMap ( 5 , range ( numPrivateTxNodes ) , ( i ) => registerIPAddress ( `${ celoEnv }-tx-nodes-private-${ i }` ) ) ; } } export async function upgradeStaticIPs ( celoEnv ) { const newTxNodeCount = parseInt ( fetchEnv ( envVar . TX_NODES ) , 10 ) ; await upgradeNodeTypeStaticIPs ( celoEnv , 'tx-nodes' , newTxNodeCount ) ; if ( useStaticIPsForGethNodes ( ) ) { const prevValidatorNodeCount = await getStatefulSetReplicas ( celoEnv , `${ celoEnv }-validators` ) ; const newValidatorNodeCount = parseInt ( fetchEnv ( envVar . VALIDATORS ) , 10 ) ; await upgradeValidatorStaticIPs ( celoEnv , prevValidatorNodeCount , newValidatorNodeCount ) ; const proxiesPerValidator = getProxiesPerValidator ( ) ; const higherValidatorCount = Math . max ( prevValidatorNodeCount , newValidatorNodeCount ) ; for ( let i = 0 ; i < higherValidatorCount ; i ++ ) { const proxyCount = proxiesPerValidator [ i ] ; await upgradeNodeTypeStaticIPs ( celoEnv , `validators-${ i }-proxy` , proxyCount ) ; } const newPrivateTxNodeCount = parseInt ( fetchEnv ( envVar . PRIVATE_TX_NODES ) , 10 ) ; await upgradeNodeTypeStaticIPs ( celoEnv , 'tx-nodes-private' , newPrivateTxNodeCount ) ; } } async function upgradeValidatorStaticIPs ( celoEnv , prevValidatorNodeCount , newValidatorNodeCount ) { const proxiesPerValidator = getProxiesPerValidator ( ) ; const higherValidatorCount = Math . max ( prevValidatorNodeCount , newValidatorNodeCount ) ; for ( let i = 0 ; i < higherValidatorCount ; i ++ ) { const ipName = `${ celoEnv }-validators-${ i }` ; let ipExists ; try { await retrieveIPAddress ( ipName ) ; ipExists = true ; } catch ( e ) { ipExists = false ; } const proxiedValidator = proxiesPerValidator [ i ] === 0 ? false : true ; if ( ipExists && proxiedValidator ) { await deleteIPAddress ( ipName ) ; } else if ( ! ipExists && ! proxiedValidator ) { await registerIPAddress ( ipName ) ; } } } async function upgradeNodeTypeStaticIPs ( celoEnv , nodeType , newNodeCount ) { const existingAddresses = await retrieveIPAddresses ( `${ celoEnv }-${ nodeType }` ) ; const desiredAddresses = range ( 0 , newNodeCount ) . map ( ( i ) => `${ celoEnv }-${ nodeType }-${ i }` ) ; const addressesToCreate = desiredAddresses . filter ( ( a ) => ! existingAddresses . includes ( a ) ) ; const addressesToDelete = existingAddresses . filter ( ( a ) => ! desiredAddresses . includes ( a ) ) ; for ( const address of addressesToCreate ) { if ( address ) { await registerIPAddress ( address ) ; } } for ( const address of addressesToDelete ) { if ( address ) { await deleteIPAddress ( address ) ; } } } export async function pollForBootnodeLoadBalancer ( celoEnv ) { if ( ! useStaticIPsForGethNodes ( ) ) { return ; } console . info ( `Poll for bootnode load balancer` ) ; let totalTime = 0 ; while ( true ) { const [ rules ] = await execCmdWithExitOnFailure ( `gcloud compute addresses describe ${ celoEnv }-bootnode --region ${ getKubernetesClusterRegion ( ) } --format="value(users.len())"` ) ; if ( parseInt ( rules , 10 ) > 0 ) { break ; } totalTime += LOAD_BALANCER_POLL_INTERVAL ; if ( totalTime > TIMEOUT_FOR_LOAD_BALANCER_POLL ) { console . error ( `\nCould not detect the bootnode's load balancer provisioning, which will likely cause the peers on the network unable to connect` ) ; process . exit ( 1 ) ; } process . stdout . write ( '.' ) ; await sleep ( LOAD_BALANCER_POLL_INTERVAL ) ; } console . info ( 'Sleeping 1 minute...' ) ; await sleep ( 1000 * 60 ) ; console . info ( `\nReset all pods now that the bootnode load balancer has provisioned` ) ; await execCmdWithExitOnFailure ( `kubectl delete pod -n ${ celoEnv } --selector=component=validators` ) ; await execCmdWithExitOnFailure ( `kubectl delete pod -n ${ celoEnv } --selector=component=tx_nodes` ) ; await execCmdWithExitOnFailure ( `kubectl delete pod -n ${ celoEnv } --selector=component=proxy` ) ; return ; } export async function deleteStaticIPs ( celoEnv ) { console . info ( `Deleting static IPs for ${ celoEnv }` ) ; const numTxNodes = parseInt ( fetchEnv ( envVar . TX_NODES ) , 10 ) ; await concurrentMap ( 5 , range ( numTxNodes ) , ( i ) => deleteIPAddress ( `${ celoEnv }-tx-nodes-${ i }` ) ) ; await deleteIPAddress ( `${ celoEnv }-bootnode` ) ; const numValidators = parseInt ( fetchEnv ( envVar . VALIDATORS ) , 10 ) ; await concurrentMap ( 5 , range ( numValidators ) , ( i ) => deleteIPAddress ( `${ celoEnv }-validators-${ i }` ) ) ; const proxiesPerValidator = getProxiesPerValidator ( ) ; for ( let valIndex = 0 ; valIndex < numValidators ; valIndex ++ ) { for ( let proxyIndex = 0 ; proxyIndex < proxiesPerValidator [ valIndex ] ; proxyIndex ++ ) { await deleteIPAddress ( getProxyName ( celoEnv , valIndex , proxyIndex ) ) ; } } const numPrivateTxNodes = parseInt ( fetchEnv ( envVar . PRIVATE_TX_NODES ) , 10 ) ; await concurrentMap ( 5 , range ( numPrivateTxNodes ) , ( i ) => deleteIPAddress ( `${ celoEnv }-tx-nodes-private-${ i }` ) ) ; } export async function deletePersistentVolumeClaims ( celoEnv , componentLabels ) { for ( const component of componentLabels ) { await deletePersistentVolumeClaimsCustomLabels ( celoEnv , 'component' , component ) ; } } export async function deletePersistentVolumeClaimsCustomLabels ( namespace , label , value ) { console . info ( `Deleting persistent volume claims for labels ${ label }=${ value } in namespace ${ namespace }` ) ; try { const [ output ] = await execCmd ( `kubectl delete pvc --selector='${ label }=${ value }' --namespace ${ namespace }` ) ; console . info ( output ) ; } catch ( error ) { console . error ( error ) ; if ( ! error . toString ( ) . includes ( 'not found' ) ) { process . exit ( 1 ) ; } } } async function helmIPParameters ( celoEnv ) { const ipAddressParameters = [ `--set geth.static_ips=${ fetchEnv ( envVar . STATIC_IPS_FOR_GETH_NODES ) }` , ] ; const numTxNodes = parseInt ( fetchEnv ( envVar . TX_NODES ) , 10 ) ; const txAddresses = await concurrentMap ( 5 , range ( numTxNodes ) , ( i ) => retrieveIPAddress ( `${ celoEnv }-tx-nodes-${ i }` ) ) ; const txNodeIpParams = setHelmArray ( 'geth.txNodesIPAddressArray' , txAddresses ) ; ipAddressParameters . push ( ... txNodeIpParams ) ; if ( useStaticIPsForGethNodes ( ) ) { ipAddressParameters . push ( `--set geth.bootnodeIpAddress=${ await retrieveBootnodeIPAddress ( celoEnv ) }` ) ; const numValidators = parseInt ( fetchEnv ( envVar . VALIDATORS ) , 10 ) ; const proxiesPerValidator = getProxiesPerValidator ( ) ; const validatorIpAddresses = [ ] ; for ( let i = 0 ; i < numValidators ; i ++ ) { if ( proxiesPerValidator [ i ] > 0 ) { validatorIpAddresses . push ( '' ) ; } else { validatorIpAddresses . push ( await retrieveIPAddress ( `${ celoEnv }-validators-${ i }` ) ) ; } } const validatorIpParams = setHelmArray ( 'geth.validatorsIPAddressArray' , validatorIpAddresses ) ; ipAddressParameters . push ( ... validatorIpParams ) ; const proxyIpAddressesPerValidator = [ ] ; let validatorIndex = 0 ; for ( const proxyCount of proxiesPerValidator ) { const proxyIpAddresses = [ ] ; for ( let i = 0 ; i < proxyCount ; i ++ ) { proxyIpAddresses . push ( await retrieveIPAddress ( getProxyName ( celoEnv , validatorIndex , i ) ) ) ; } const listOfProxyIpAddresses = proxyIpAddresses . join ( '/' ) ; proxyIpAddressesPerValidator . push ( listOfProxyIpAddresses ) ; validatorIndex ++ ; } const proxyIpAddressesParams = setHelmArray ( 'geth.proxyIPAddressesPerValidatorArray' , proxyIpAddressesPerValidator ) ; ipAddressParameters . push ( ... proxyIpAddressesParams ) ; const numPrivateTxNodes = parseInt ( fetchEnv ( envVar . PRIVATE_TX_NODES ) , 10 ) ; const privateTxAddresses = await concurrentMap ( 5 , range ( numPrivateTxNodes ) , ( i ) => retrieveIPAddress ( `${ celoEnv }-tx-nodes-private-${ i }` ) ) ; const privateTxAddressParameters = privateTxAddresses . map ( ( address , i ) => `--set geth.private_tx_nodes_${ i }IpAddress=${ address }` ) ; ipAddressParameters . push ( ... privateTxAddressParameters ) ; const listOfPrivateTxNodeAddresses = privateTxAddresses . join ( ',' ) ; ipAddressParameters . push ( `--set geth.private_tx_node_ip_addresses='{${ listOfPrivateTxNodeAddresses }}'` ) ; } return ipAddressParameters ; } async function helmParameters ( celoEnv , useExistingGenesis ) { const valueFilePath = `/tmp/${ celoEnv }-testnet-values.yaml` ; await saveHelmValuesFile ( celoEnv , valueFilePath , useExistingGenesis , false ) ; const gethMetricsOverrides = fetchEnvOrFallback ( 'GETH_ENABLE_METRICS' , 'false' ) === 'true' ? [ `--set metrics="true"` , `--set pprof.enabled="true"` , `--set pprof.path="/debug/metrics/prometheus"` , `--set pprof.port="6060"` , ] : [ `--set metrics="false"` , `--set pprof.enabled="false"` ] ; const useMyCelo = stringToBoolean ( fetchEnvOrFallback ( envVar . GETH_USE_MYCELO , 'false' ) ) ; await createAndPushGenesis ( celoEnv , ! useExistingGenesis , useMyCelo ) ; const bootnodeOverwritePkey = fetchEnvOrFallback ( envVar . GETH_BOOTNODE_OVERWRITE_PKEY , '' ) !== '' ? [ `--set geth.overwriteBootnodePrivateKey="true"` , `--set geth.bootnodePrivateKey="${ fetchEnv ( envVar . GETH_BOOTNODE_OVERWRITE_PKEY ) }"` , ] : [ `--set geth.overwriteBootnodePrivateKey="false"` ] ; const defaultDiskSize = fetchEnvOrFallback ( envVar . NODE_DISK_SIZE_GB , '10' ) ; const privateTxNodeDiskSize = fetchEnvOrFallback ( envVar . PRIVATE_NODE_DISK_SIZE_GB , defaultDiskSize ) ; return [ `-f ${ valueFilePath }` , `--set bootnode.image.repository=${ fetchEnv ( 'GETH_BOOTNODE_DOCKER_IMAGE_REPOSITORY' ) }` , `--set bootnode.image.tag=${ fetchEnv ( 'GETH_BOOTNODE_DOCKER_IMAGE_TAG' ) }` , `--set celotool.image.repository=${ fetchEnv ( 'CELOTOOL_DOCKER_IMAGE_REPOSITORY' ) }` , `--set celotool.image.tag=${ fetchEnv ( 'CELOTOOL_DOCKER_IMAGE_TAG' ) }` , `--set domain.name=${ fetchEnv ( 'CLUSTER_DOMAIN_NAME' ) }` , `--set genesis.useGenesisFileBase64="false"` , `--set genesis.network=${ celoEnv }` , `--set genesis.networkId=${ fetchEnv ( envVar . NETWORK_ID ) }` , `--set genesis.epoch_size=${ fetchEnv ( envVar . EPOCH ) }` , `--set geth.verbosity=${ fetchEnvOrFallback ( 'GETH_VERBOSITY' , '4' ) }` , `--set geth.vmodule=${ fetchEnvOrFallback ( 'GETH_VMODULE' , '' ) }` , `--set geth.resources.requests.cpu=${ fetchEnv ( 'GETH_NODE_CPU_REQUEST' ) }` , `--set geth.resources.requests.memory=${ fetchEnv ( 'GETH_NODE_MEMORY_REQUEST' ) }` , `--set geth.image.repository=${ fetchEnv ( 'GETH_NODE_DOCKER_IMAGE_REPOSITORY' ) }` , `--set geth.image.tag=${ fetchEnv ( 'GETH_NODE_DOCKER_IMAGE_TAG' ) }` , `--set geth.validators="${ fetchEnv ( 'VALIDATORS' ) }"` , `--set geth.secondaries="${ fetchEnvOrFallback ( 'SECONDARIES' , '0' ) }"` , `--set geth.use_gstorage_data=${ fetchEnvOrFallback ( 'USE_GSTORAGE_DATA' , 'false' ) }` , `--set geth.gstorage_data_bucket=${ fetchEnvOrFallback ( 'GSTORAGE_DATA_BUCKET' , '' ) }` , `--set geth.faultyValidators="${ fetchEnvOrFallback ( 'FAULTY_VALIDATORS' , '0' ) }"` , `--set geth.faultyValidatorType="${ fetchEnvOrFallback ( 'FAULTY_VALIDATOR_TYPE' , '0' ) }"` , `--set geth.fullnodeCheckBlockAge=${ fetchEnvOrFallback ( envVar . FULL_NODE_READINESS_CHECK_BLOCK_AGE , `${ isProduction ( ) }` ) }` , `--set geth.tx_nodes="${ fetchEnv ( 'TX_NODES' ) }"` , `--set geth.private_tx_nodes="${ fetchEnv ( envVar . PRIVATE_TX_NODES ) }"` , `--set geth.ssd_disks="${ fetchEnvOrFallback ( envVar . GETH_NODES_SSD_DISKS , 'true' ) }"` , `--set geth.account.secret="${ fetchEnv ( 'GETH_ACCOUNT_SECRET' ) }"` , `--set geth.ping_ip_from_packet=${ fetchEnvOrFallback ( 'PING_IP_FROM_PACKET' , 'false' ) }` , `--set geth.in_memory_discovery_table=${ fetchEnvOrFallback ( 'IN_MEMORY_DISCOVERY_TABLE' , 'false' ) }` , `--set geth.diskSizeGB=${ defaultDiskSize }` , `--set geth.privateTxNodediskSizeGB=${ privateTxNodeDiskSize }` , `--set mnemonic="${ fetchEnv ( 'MNEMONIC' ) }"` , ... setHelmArray ( 'geth.proxiesPerValidator' , getProxiesPerValidator ( ) ) , ... gethMetricsOverrides , ... bootnodeOverwritePkey , ... rollingUpdateHelmVariables ( ) , ... ( await helmIPParameters ( celoEnv ) ) , ] ; } async function helmCommand ( command , pipeOutput = false ) { if ( isCelotoolVerbose ( ) && ! command . startsWith ( 'helm diff' ) ) { command += ' --debug' ; } await execCmdWithExitOnFailure ( command , { } , pipeOutput ) ; } function buildHelmChartDependencies ( chartDir ) { console . info ( `Building any chart dependencies...` ) ; return helmCommand ( `helm dep build ${ chartDir }` ) ; } export async function installHelmDiffPlugin ( ) { try { await execCmd ( `helm diff version` , { } , false ) ; } catch ( error ) { console . info ( `Installing helm-diff plugin...` ) ; await execCmdWithExitOnFailure ( `helm plugin install https://github.com/databus23/helm-diff` ) ; } } function valuesOverrideArg ( chartDir , filename ) { if ( filename === undefined ) { return '' ; } return `-f ${ chartDir }/${ filename }` ; } export async function installGenericHelmChart ( namespace , releaseName , chartDir , parameters , buildDependencies = true , valuesOverrideFile ? ) { if ( buildDependencies ) { await buildHelmChartDependencies ( chartDir ) ; } if ( isCelotoolHelmDryRun ( ) ) { console . info ( `This would deploy chart ${ chartDir } with release name ${ releaseName } in namespace ${ namespace } with parameters:` ) ; console . info ( parameters ) ; if ( valuesOverrideFile !== undefined ) { console . info ( `And with values override: ${ valuesOverrideFile }` ) ; } } else { console . info ( `Installing helm release ${ releaseName }` ) ; const valuesOverride = valuesOverrideArg ( chartDir , valuesOverrideFile ) ; await helmCommand ( `helm upgrade --install -f ${ chartDir }/values.yaml ${ valuesOverride } ${ releaseName } ${ chartDir } --namespace ${ namespace } ${ parameters . join ( ' ' ) }` ) ; } } export async function upgradeGenericHelmChart ( namespace , releaseName , chartDir , parameters , valuesOverrideFile ? ) { await buildHelmChartDependencies ( chartDir ) ; const valuesOverride = valuesOverrideArg ( chartDir , valuesOverrideFile ) ; if ( isCelotoolHelmDryRun ( ) ) { console . info ( `Simulating the upgrade of helm release ${ releaseName }. No output means no change in the helm release` ) ; await installHelmDiffPlugin ( ) ; await helmCommand ( `helm diff upgrade -C 5 -f ${ chartDir }/values.yaml ${ valuesOverride } ${ releaseName } ${ chartDir } --namespace ${ namespace } ${ parameters . join ( ' ' ) }` , true ) ; } else { console . info ( `Upgrading helm release ${ releaseName }` ) ; await helmCommand ( `helm upgrade -f ${ chartDir }/values.yaml ${ valuesOverride } ${ releaseName } ${ chartDir } --namespace ${ namespace } ${ parameters . join ( ' ' ) }` ) ; console . info ( `Upgraded helm release ${ releaseName } successful` ) ; } } export async function getConfigMapHashes ( celoEnv , releaseName , chartDir , parameters , action , valuesOverrideFile ? ) { const valuesOverride = valuesOverrideArg ( chartDir , valuesOverrideFile ) ; const [ output ] = await execCmd ( `helm ${ action } -f ${ chartDir }/values.yaml ${ valuesOverride } ${ releaseName } ${ chartDir } --namespace ${ celoEnv } ${ parameters . join ( ' ' ) } --dry-run` , { } , false , false ) ; return output . split ( '---' ) . filter ( ( section ) => { return /kind: ConfigMap/ . exec ( section ) ; } ) . reduce < Record < string , string > > ( ( configHashes , section ) => { const matchSource = /Source: (.*)/ . exec ( section ) ; if ( matchSource === null ) { throw new Error ( 'Can not extract Source from config section' ) ; } configHashes [ matchSource [ 1 ] ] = stringHash ( section ) . toString ( ) ; return configHashes ; } , { } ) ; } export function isCelotoolVerbose ( ) { return process . env . CELOTOOL_VERBOSE === 'true' ; } export function isCelotoolHelmDryRun ( ) { return process . env . CELOTOOL_HELM_DRY_RUN === 'true' ; } export function exitIfCelotoolHelmDryRun ( ) { if ( isCelotoolHelmDryRun ( ) ) { console . error ( 'Option --helmdryrun is not allowed for this command. Exiting.' ) ; process . exit ( 1 ) ; } } export async function removeGenericHelmChart ( releaseName , namespace ) { console . info ( `Deleting helm chart ${ releaseName } from namespace ${ namespace }` ) ; try { await execCmd ( `helm uninstall --namespace ${ namespace } ${ releaseName }` ) ; } catch ( error ) { console . error ( error ) ; } } function getExtraValuesFile ( celoEnv ) { const extraValuesFile = fs . existsSync ( `${ TESTNET_CHART_DIR }/values-${ celoEnv }.yaml` ) ? `values-${ celoEnv }.yaml` : undefined ; return extraValuesFile ; } export async function installHelmChart ( celoEnv , useExistingGenesis ) { await failIfSecretMissing ( BACKUP_GCS_SECRET_NAME , 'default' ) ; await copySecret ( BACKUP_GCS_SECRET_NAME , 'default' , celoEnv ) ; const extraValuesFile = getExtraValuesFile ( celoEnv ) ; return installGenericHelmChart ( celoEnv , celoEnv , TESTNET_CHART_DIR , await helmParameters ( celoEnv , useExistingGenesis ) , true , extraValuesFile ) ; } export async function upgradeHelmChart ( celoEnv , useExistingGenesis ) { console . info ( `Upgrading helm release ${ celoEnv }` ) ; const parameters = await helmParameters ( celoEnv , useExistingGenesis ) ; const extraValuesFile = getExtraValuesFile ( celoEnv ) ; await upgradeGenericHelmChart ( celoEnv , celoEnv , TESTNET_CHART_DIR , parameters , extraValuesFile ) ; } export async function resetAndUpgradeHelmChart ( celoEnv , useExistingGenesis ) { const txNodesSetName = `${ celoEnv }-tx-nodes` ; const validatorsSetName = `${ celoEnv }-validators` ; const bootnodeName = `${ celoEnv }-bootnode` ; const privateTxNodesSetname = `${ celoEnv }-tx-nodes-private` ; const persistentVolumeClaimsLabels = [ 'validators' , 'tx_nodes' , 'proxy' , 'tx_nodes_private' ] ; if ( isCelotoolHelmDryRun ( ) ) { await upgradeHelmChart ( celoEnv , useExistingGenesis ) ; } else { await scaleResource ( celoEnv , 'StatefulSet' , txNodesSetName , 0 ) ; await scaleResource ( celoEnv , 'StatefulSet' , validatorsSetName , 0 ) ; await scaleResource ( celoEnv , 'StatefulSet' , privateTxNodesSetname , 0 , true ) ; await scaleProxies ( celoEnv , 0 ) ; await scaleResource ( celoEnv , 'Deployment' , bootnodeName , 0 ) ; await deletePersistentVolumeClaims ( celoEnv , persistentVolumeClaimsLabels ) ; await sleep ( 10000 ) ; await upgradeHelmChart ( celoEnv , useExistingGenesis ) ; await sleep ( 10000 ) ; const numValdiators = parseInt ( fetchEnv ( envVar . VALIDATORS ) , 10 ) ; const numTxNodes = parseInt ( fetchEnv ( envVar . TX_NODES ) , 10 ) ; const numPrivateTxNodes = parseInt ( fetchEnv ( envVar . PRIVATE_TX_NODES ) , 10 ) ; await scaleResource ( celoEnv , 'StatefulSet' , txNodesSetName , numTxNodes ) ; await scaleResource ( celoEnv , 'StatefulSet' , validatorsSetName , numValdiators ) ; await scaleResource ( celoEnv , 'StatefulSet' , privateTxNodesSetname , numPrivateTxNodes ) ; await scaleProxies ( celoEnv ) ; await scaleResource ( celoEnv , 'Deployment' , bootnodeName , 1 ) ; } } async function scaleProxies ( celoEnv , replicas ? ) { if ( replicas !== undefined ) { const statefulsetNames = await getProxyStatefulsets ( celoEnv ) ; for ( const name of statefulsetNames ) { await scaleResource ( celoEnv , 'StatefulSet' , name , replicas ) ; } } else { const proxiesPerValidator = getProxiesPerValidator ( ) ; let validatorIndex = 0 ; for ( const proxyCount of proxiesPerValidator ) { await scaleResource ( celoEnv , 'StatefulSet' , `${ celoEnv }-validators-${ validatorIndex }-proxy` , proxyCount , true ) ; validatorIndex ++ ; } } } async function getProxyStatefulsets ( celoEnv ) { const [ output ] = await execCmd ( `kubectl get statefulsets --selector=component=proxy --no-headers -o custom-columns=":metadata.name" -n ${ celoEnv }` ) ; if ( ! output ) { return [ ] ; } return output . split ( '\n' ) . filter ( ( name ) => name ) ; } export async function removeHelmRelease ( celoEnv ) { return removeGenericHelmChart ( celoEnv , celoEnv ) ; } export function makeHelmParameters ( map ) { return entries ( map ) . map ( ( [ key , value ] ) => `--set ${ key }=${ value }` ) ; } export function setHelmArray ( paramName , arr ) { return arr . map ( ( value , i ) => `--set ${ paramName }[${ i }]="${ value }"` ) ; } export async function deleteFromCluster ( celoEnv ) { await removeHelmRelease ( celoEnv ) ; console . info ( `Deleting namespace ${ celoEnv }` ) ; await execCmdWithExitOnFailure ( `kubectl delete namespace ${ celoEnv }` ) ; } function useStaticIPsForGethNodes ( ) { return fetchEnv ( envVar . STATIC_IPS_FOR_GETH_NODES ) === 'true' ; } export async function checkHelmVersion ( ) { const requiredMinHelmVersion = '3.4' ; const helmVersionCmd = `helm version --template '{{ .Version }}'` ; const localHelmVersion = ( await execCmdWithExitOnFailure ( helmVersionCmd ) ) [ 0 ] . replace ( '^v' , '' ) ; const helmOK = compareVersions . compare ( localHelmVersion , requiredMinHelmVersion , '>=' ) ; if ( helmOK ) { return true ; } else { console . error ( `Error checking local helm version. Minimum Helm version required ${ requiredMinHelmVersion }` ) ; process . exit ( 1 ) ; } } function rollingUpdateHelmVariables ( ) { return [ `--set updateStrategy.validators.rollingUpdate.partition=${ fetchEnvOrFallback ( envVar . VALIDATORS_ROLLING_UPDATE_PARTITION , '0' ) }` , `--set updateStrategy.secondaries.rollingUpdate.partition=${ fetchEnvOrFallback ( envVar . SECONDARIES_ROLLING_UPDATE_PARTITION , '0' ) }` , `--set updateStrategy.proxy.rollingUpdate.partition=${ fetchEnvOrFallback ( envVar . PROXY_ROLLING_UPDATE_PARTITION , '0' ) }` , `--set updateStrategy.tx_nodes.rollingUpdate.partition=${ fetchEnvOrFallback ( envVar . TX_NODES_ROLLING_UPDATE_PARTITION , '0' ) }` , `--set updateStrategy.tx_nodes_private.rollingUpdate.partition=${ fetchEnvOrFallback ( envVar . TX_NODES_PRIVATE_ROLLING_UPDATE_PARTITION , '0' ) }` , ] ; } export async function saveHelmValuesFile ( celoEnv , valueFilePath , useExistingGenesis , skipGenesisValue = false ) { const genesisContent = useExistingGenesis ? await getGenesisBlockFromGoogleStorage ( celoEnv ) : generateGenesisFromEnv ( ) ; const enodes = await getEnodesWithExternalIPAddresses ( celoEnv ) ; let valueFileContent = `
staticnodes:
  staticnodesBase64: ${ Buffer . from ( JSON . stringify ( enodes ) ) . toString ( 'base64' ) }
` ; if ( ! skipGenesisValue ) { valueFileContent += `
  genesis:
    genesisFileBase64: ${ Buffer . from ( genesisContent ) . toString ( 'base64' ) }
` ; } fs . writeFileSync ( valueFilePath , valueFileContent ) ; } const celoBlockchainDir = path . join ( os . tmpdir ( ) , 'celo-blockchain-celotool' ) ; export async function createAndPushGenesis ( celoEnv , reset , useMyCelo ) { let genesis = '' ; try { genesis = await getGenesisBlockFromGoogleStorage ( celoEnv ) ; } catch { console . debug ( `Genesis file not found in GCP. Creating a new one` ) ; } if ( genesis === '' || reset === true ) { genesis = useMyCelo ? await generateMyCeloGenesis ( ) : generateGenesisFromEnv ( ) ; } if ( ! isCelotoolHelmDryRun ( ) ) { await uploadGenesisBlockToGoogleStorage ( celoEnv , genesis ) ; } } async function generateMyCeloGenesis ( ) { await spawnCmd ( 'rm' , [ '-rf' , celoBlockchainDir ] , { silent : true } ) ; fs . mkdirSync ( celoBlockchainDir ) ; const gethTag = fetchEnvOrFallback ( envVar . GETH_MYCELO_COMMIT , '' ) !== '' ? fetchEnv ( envVar . GETH_MYCELO_COMMIT ) : fetchEnv ( envVar . GETH_NODE_DOCKER_IMAGE_TAG ) ; const celoBlockchainVersion = gethTag . includes ( '.' ) ? `v${ gethTag }` : gethTag ; await checkoutGethRepo ( celoBlockchainVersion , celoBlockchainDir ) ; await buildGethAll ( celoBlockchainDir ) ; const myceloBinary = path . join ( celoBlockchainDir , 'build/bin/mycelo' ) ; const myceloGenesisConfigArgs = [ 'genesis-config' , '--template' , 'monorepo' , '--mnemonic' , fetchEnv ( envVar . MNEMONIC ) , '--validators' , fetchEnv ( envVar . VALIDATORS ) , '--dev.accounts' , fetchEnv ( envVar . LOAD_TEST_CLIENTS ) , '--blockperiod' , fetchEnv ( envVar . BLOCK_TIME ) , '--epoch' , fetchEnv ( envVar . EPOCH ) , '--blockgaslimit' , '20000000' , ] ; await spawnCmdWithExitOnFailure ( myceloBinary , myceloGenesisConfigArgs , { silent : false , cwd : celoBlockchainDir , } ) ; const myceloGenesisFromConfigArgs = [ 'genesis-from-config' , celoBlockchainDir , '--buildpath' , path . join ( monorepoRoot , 'packages/protocol/build/contracts' ) , ] ; await spawnCmdWithExitOnFailure ( myceloBinary , myceloGenesisFromConfigArgs , { silent : false , cwd : celoBlockchainDir , } ) ; const genesisPath = path . join ( celoBlockchainDir , 'genesis.json' ) ; const genesisContent = fs . readFileSync ( genesisPath ) . toString ( ) ; await spawnCmd ( 'rm' , [ '-rf' , celoBlockchainDir ] , { silent : true } ) ; return genesisContent ; }