import { StaticNodeUtils } from '@celo/network-utils' ; import { GenesisBlocksGoogleStorageBucketName } from '@celo/network-utils/lib/genesis-block-utils' ; import { Storage } from '@google-cloud/storage' ; import * as fs from 'fs' ; import fetch from 'node-fetch' ; import * as path from 'path' ; import { retryCmd } from '../lib/utils' ; import { execCmdWithExitOnFailure } from './cmd-utils' ; import { getGenesisGoogleStorageUrl } from './endpoints' ; import { envVar , fetchEnvOrFallback , getEnvFile } from './env-utils' ; import { ensureAuthenticatedGcloudAccount } from './gcloud_utils' ; import { getBootnodeEnode , getEnodesWithExternalIPAddresses } from './geth' ; const genesisBlocksBucketName = GenesisBlocksGoogleStorageBucketName ; const staticNodesBucketName = StaticNodeUtils . getStaticNodesGoogleStorageBucketName ( ) ; const envBucketName = 'env_config_files' ; const bootnodesBucketName = 'env_bootnodes' ; export async function uploadTestnetInfoToGoogleStorage ( networkName ) { await uploadTestnetStaticNodesToGoogleStorage ( networkName ) ; await uploadBootnodeToGoogleStorage ( networkName ) ; await uploadEnvFileToGoogleStorage ( networkName ) ; } export async function uploadGenesisBlockToGoogleStorage ( networkName , genesis ) { console . info ( `\nUploading genesis block for ${ networkName } to Google cloud storage` ) ; console . debug ( `Genesis block is ${ genesis } \n` ) ; await uploadDataToGoogleStorage ( genesis , genesisBlocksBucketName , networkName , true , 'application/json' ) ; } export async function getGenesisBlockFromGoogleStorage ( networkName ) { const resp = await fetch ( getGenesisGoogleStorageUrl ( networkName ) ) ; return JSON . stringify ( await resp . json ( ) ) ; } export async function uploadTestnetStaticNodesToGoogleStorage ( networkName ) { console . info ( `\nUploading static nodes for ${ networkName } to Google cloud storage...` ) ; const nodesData = await retryCmd ( ( ) => getEnodesWithExternalIPAddresses ( networkName ) ) ; if ( nodesData === null ) { throw new Error ( 'Fail to get static nodes information' ) ; } return uploadStaticNodesToGoogleStorage ( networkName , nodesData ) ; } export async function uploadStaticNodesToGoogleStorage ( fileName , enodes ) { const json = JSON . stringify ( enodes ) ; console . debug ( `${ fileName } static nodes are ${ json }\n` ) ; await uploadDataToGoogleStorage ( json , staticNodesBucketName , fileName , true , 'application/json' ) ; } export async function uploadBootnodeToGoogleStorage ( networkName ) { console . info ( `\nUploading bootnode for ${ networkName } to Google Cloud Storage...` ) ; const [ bootnodeEnode ] = await retryCmd ( ( ) => getBootnodeEnode ( networkName ) ) ; if ( ! bootnodeEnode ) { throw new Error ( 'Failed to get bootnode enode' ) ; } console . info ( 'Bootnode enode:' , bootnodeEnode ) ; await uploadDataToGoogleStorage ( bootnodeEnode , bootnodesBucketName , networkName , true , 'text/plain' ) ; } export async function uploadEnvFileToGoogleStorage ( networkName ) { const envFileName = getEnvFile ( networkName ) ; const userInfo = `${ await getGoogleCloudUserInfo ( ) }` ; const repo = await getGitRepoName ( ) ; const commitHash = await getCommitHash ( ) ; console . info ( `\nUploading Env file ${ envFileName } for network ${ networkName } to Google cloud storage: ` + `gs://${ envBucketName }/${ networkName }` ) ; const envFileData = fs . readFileSync ( getEnvFile ( networkName ) ) . toString ( ) ; const metaData = `# .env file for network "${ networkName }"\n` + `# Last modified by "${ userInfo }"\n` + `# Last modified on on ${ Date ( ) }\n` + `# Base commit: "https://github.com/${ repo }/commit/${ commitHash }"\n` ; const fullData = metaData + '\n' + envFileData ; await uploadDataToGoogleStorage ( fullData , envBucketName , networkName , false , 'text/plain' ) ; } async function getGoogleCloudUserInfo ( ) { const cmd = 'gcloud config get-value account' ; const stdout = ( await execCmdWithExitOnFailure ( cmd ) ) [ 0 ] ; return stdout . trim ( ) ; } async function getGitRepoName ( ) { const cmd = 'git config --get remote.origin.url' ; let stdout = '' ; try { stdout = ( await execCmdWithExitOnFailure ( cmd ) ) [ 0 ] . trim ( ) ; stdout = stdout . split ( ':' ) [ 1 ] ; if ( stdout . endsWith ( '.git' ) ) { stdout = stdout . substring ( 0 , stdout . length - '.git' . length ) ; } } catch ( error ) { stdout = 'celo-monorepo' ; } return stdout ; } async function getCommitHash ( ) { try { const cmd = 'git show | head -n 1' ; const stdout = ( await execCmdWithExitOnFailure ( cmd ) ) [ 0 ] ; return stdout . split ( ' ' ) [ 1 ] . trim ( ) ; } catch ( error ) { return 'no-commmit-hash' ; } } export function uploadDataToGoogleStorage ( data , googleStorageBucketName , googleStorageFileName , makeFileWorldReadable , contentType ) { const localTmpFilePath = `/tmp/${ googleStorageBucketName }-${ googleStorageFileName }` ; fs . mkdirSync ( path . dirname ( localTmpFilePath ) , { recursive : true , } ) ; fs . writeFileSync ( localTmpFilePath , data ) ; return uploadFileToGoogleStorage ( localTmpFilePath , googleStorageBucketName , googleStorageFileName , makeFileWorldReadable , contentType ) ; } export async function uploadFileToGoogleStorage ( localFilePath , googleStorageBucketName , googleStorageFileName , makeFileWorldReadable , contentType ) { await ensureAuthenticatedGcloudAccount ( ) ; const storage = new Storage ( ) ; await storage . bucket ( googleStorageBucketName ) . upload ( localFilePath , { destination : googleStorageFileName , contentType , metadata : { cacheControl : 'private' , } , } ) ; if ( makeFileWorldReadable ) { await storage . bucket ( googleStorageBucketName ) . file ( googleStorageFileName ) . acl . add ( { entity : 'allUsers' , role : storage . acl . READER_ROLE , } ) ; } } export function getProxiesPerValidator ( ) { const arr = [ ] ; const valProxyCountsStr = fetchEnvOrFallback ( envVar . VALIDATOR_PROXY_COUNTS , '' ) ; const splitValProxyCountStrs = valProxyCountsStr . split ( ',' ) . filter ( ( counts ) => counts ) ; for ( const valProxyCount of splitValProxyCountStrs ) { const [ valCountStr , proxyCountStr ] = valProxyCount . split ( ':' ) ; const valCount = parseInt ( valCountStr , 10 ) ; const proxyCount = parseInt ( proxyCountStr , 10 ) ; for ( let i = 0 ; i < valCount ; i ++ ) { arr . push ( proxyCount ) ; } } return arr ; } export function getProxyName ( celoEnv , validatorIndex , proxyIndex ) { return `${ celoEnv }-validators-${ validatorIndex }-proxy-${ proxyIndex }` ; }