import { waitFor } from "@adpt/utils" ; import Docker = require ( "dockerode" ) ; import * as jsYaml from "js-yaml" ; import * as ld from "lodash" ; import moment from "moment" ; import { URL } from "url" ; import * as util from "util" ; import { addToNetwork , createNetwork , dockerExec , dockerPull , getNetwork , getSelfContainer , removeFromNetwork , } from "./dockerutils" ; const stripAnsi = require ( "strip-ansi" ) ; export interface MinikubeInfo { docker : Docker ; dockerHost : string ; dockerIP : string ; container : Docker . Container ; hostname : string ; network : Docker . Network | null ; kubeconfig : object ; stop : ( ) => Promise < void > ; exec : ( command : string [ ] ) => Promise < string > ; } async function getKubeconfig ( _docker , container , hostname , containerAlias ? , portAlias ? ) { if ( ! containerAlias ) containerAlias = hostname ; const configYAML = await dockerExec ( container , [ "cat" , "/kubeconfig" ] ) ; const kubeconfig = jsYaml . safeLoad ( configYAML ) ; if ( ! kubeconfig || typeof kubeconfig !== "object" ) { throw new Error ( `Invalid kubeconfig\n ${ configYAML }\n\n${ util . inspect ( kubeconfig ) }` ) ; } const clusters = ( kubeconfig as any ) . clusters ; if ( ! ld . isArray ( clusters ) ) { throw new Error ( `Invalid kubeconfig\n ${ configYAML }\n\n${ util . inspect ( kubeconfig ) }` ) ; } for ( const cluster of clusters ) { const url = new URL ( cluster . cluster . server ) ; url . host = containerAlias ; if ( portAlias ) url . port = portAlias . toString ( ) ; cluster . cluster . server = url . href ; } return kubeconfig ; } async function runMinikubeContainer ( docker , containerName , networkName ) { const imageName = "unboundedsystems/k3s-dind" ; const imageTag = "1.18.4-k3s1" ; const image = `${ imageName }:${ imageTag }` ; const opts = { name : containerName , AttachStdin : false , AttachStdout : false , AttachStderr : false , Hostname : containerName , Tty : false , OpenStdin : false , StdinOnce : false , HostConfig : { AutoRemove : true , Privileged : true } , Env : [ ] , Image : image , Volumes : { } , } ; if ( networkName ) { opts . HostConfig ! . NetworkMode = networkName ; opts . NetworkingConfig = { EndpointsConfig : { [ networkName ] : { } } } ; } else { opts . HostConfig ! . PublishAllPorts = true ; } await dockerPull ( docker , image , "      " ) ; const container = await docker . createContainer ( opts ) ; if ( networkName ) { const bridge = docker . getNetwork ( "bridge" ) ; await addToNetwork ( container , bridge ) ; } await container . start ( ) ; return container ; } async function waitForKubeConfig ( docker , container , hostname , containerAlias , portAlias ) { let config ; await waitFor ( 100 , 1 , "Timed out waiting for kubeconfig" , async ( ) => { try { await dockerExec ( container , [ "cat" , "/minikube_startup_complete" ] ) ; config = await getKubeconfig ( docker , container , hostname , containerAlias , portAlias ) ; return true ; } catch ( err ) { if ( /exited with error/ . test ( err . message ) || /Invalid kubeconfig/ . test ( err . message ) ) return false ; throw err ; } } ) ; return config ; } async function waitForMiniKube ( container ) { await waitFor ( 100 , 1 , "Timed out waiting for Minikube" , async ( ) => { try { const statusColor = await dockerExec ( container , [ "kubectl" , "cluster-info" ] ) ; const status = stripAnsi ( statusColor ) as string ; if ( ! /^Kubernetes master is running at/ . test ( status ) ) { return false ; } const accts = await dockerExec ( container , [ "kubectl" , "get" , "serviceaccounts" ] ) ; if ( ! /^default\s/m . test ( accts ) ) return false ; const systemPods = await dockerExec ( container , [ "kubectl" , "get" , "pods" , "--namespace=kube-system" , ] ) ; if ( ! systemPods ) return false ; const lines = systemPods . split ( "\n" ) ; if ( lines . length < 3 ) return false ; return true ; } catch ( err ) { if ( ! /exited with error/ . test ( err . message ) ) throw err ; } return false ; } ) ; } function secondsSince ( start ) { return ( Date . now ( ) - start ) / 1000 ; } function getHostPort ( info , p ) { const list = info . NetworkSettings . Ports [ `${ p }/tcp` ] ; if ( ! list || list . length === 0 ) throw new Error ( `Port ${ p } is not bound` ) ; const hp = parseInt ( list [ 0 ] . HostPort , 10 ) ; if ( isNaN ( hp ) ) throw new Error ( `HostPort is not an integer` ) ; return hp ; } export async function startTestMinikube ( ) { const stops = [ ] ; async function stop ( ) { for ( const f of stops ) { await f ( ) ; } } const startTime = Date . now ( ) ; let kubeconfig ; try { const docker = new Docker ( ) ; const self = await getSelfContainer ( docker ) ; let container ; let network = null ; let hostname ; let apiPort = 8443 ; let dockerPort = 2375 ; let info ; if ( process . env . ADAPT_TEST_K8S ) { hostname = process . env . ADAPT_TEST_K8S ; container = docker . getContainer ( hostname ) ; network = await getNetwork ( docker , container ) ; kubeconfig = await getKubeconfig ( docker , container , hostname ) ; } else { console . log ( `    Starting Minikube` ) ; const tstamp = moment ( ) . format ( "MMDD-HHmm-ss-SSSSSS" ) ; hostname = `test-k8s-${ process . pid }-${ tstamp }` ; if ( self ) { network = await createNetwork ( docker , hostname ) ; if ( network . id === undefined ) throw new Error ( "Network id was undefined!" ) ; stops . unshift ( async ( ) => network && network . remove ( ) ) ; } container = await runMinikubeContainer ( docker , hostname , network && network . id ) ; stops . unshift ( async ( ) => container . stop ( ) ) ; const hostAlias = self ? undefined : "localhost" ; if ( ! self ) { info = await container . inspect ( ) ; apiPort = getHostPort ( info , apiPort ) ; dockerPort = getHostPort ( info , dockerPort ) ; } kubeconfig = await waitForKubeConfig ( docker , container , hostname , hostAlias , apiPort ) ; if ( hostAlias ) hostname = hostAlias ; const configTime = secondsSince ( startTime ) ; console . log ( `    Got kubeconfig (${ configTime } seconds)` ) ; } if ( ! kubeconfig ) throw new Error ( "Internal Error: should be unreachable" ) ; await waitForMiniKube ( container ) ; const totalTime = secondsSince ( startTime ) ; console . log ( `\n    Minikube ready in ${ totalTime } seconds` ) ; if ( self && network ) await addToNetwork ( self , network ) ; if ( ! process . env . ADAPT_TEST_K8S ) { stops . unshift ( async ( ) => { if ( self && network ) await removeFromNetwork ( self , network ) ; } ) ; } if ( ! info ) info = await container . inspect ( ) ; const dockerIP = info . NetworkSettings . IPAddress ; const dockerHost = `tcp://${ hostname }:${ dockerPort }` ; const exec = ( command ) => dockerExec ( container , command ) ; return { container , docker , dockerHost , dockerIP , exec , hostname , kubeconfig , network , stop , } ; } catch ( e ) { await stop ( ) ; throw e ; } } export async function stopTestMinikube ( info ) { await info . stop ( ) ; }