import { promisify } from "util" ; import { dirname , join , resolve } from "path" ; import { exec as exec0 , spawn } from "child_process" ; import spawnAsync from "await-spawn" ; import * as fs from "fs" ; import { projects , root } from "@cocalc/backend/data" ; import { is_valid_uuid_string } from "@cocalc/util/misc" ; import { callback2 } from "@cocalc/util/async-utils" ; import getLogger from "@cocalc/backend/logger" ; import { CopyOptions , ProjectState , ProjectStatus } from "./base" ; import { getUid } from "@cocalc/backend/misc" ; import base_path from "@cocalc/backend/base-path" ; import { db } from "@cocalc/database" ; const winston = getLogger ( "project-control:util" ) ; export const mkdir = promisify ( fs . mkdir ) ; const readFile = promisify ( fs . readFile ) ; const stat = promisify ( fs . stat ) ; const copyFile = promisify ( fs . copyFile ) ; const rm = promisify ( fs . rm ) ; export async function chown ( path , uid ) { await promisify ( fs . chown ) ( path , uid , uid ) ; } export function dataPath ( HOME ) { return join ( HOME , ".smc" ) ; } export function homePath ( project_id ) { return projects . replace ( "[project_id]" , project_id ) ; } export function getUsername ( project_id ) { return project_id . split ( "-" ) . join ( "" ) ; } function pidIsRunning ( pid ) { try { process . kill ( pid , 0 ) ; return true ; } catch ( _err ) { return false ; } } function pidFile ( HOME ) { return join ( dataPath ( HOME ) , "project.pid" ) ; } export async function getProjectPID ( HOME ) { return parseInt ( ( await readFile ( pidFile ( HOME ) ) ) . toString ( ) ) ; } export async function isProjectRunning ( HOME ) { try { const pid = await getProjectPID ( HOME ) ; return pidIsRunning ( pid ) ; } catch ( err ) { return false ; } } export async function setupDataPath ( HOME , uid ? ) { const data = dataPath ( HOME ) ; winston . debug ( `setup "${ data }"...` ) ; await rm ( data , { recursive : true , force : true } ) ; await mkdir ( data ) ; if ( uid != null ) { await chown ( data , uid ) ; } } export async function launchProjectDaemon ( env , uid ? ) { winston . debug ( `launching project daemon at "${ env . HOME }"...` ) ; winston . debug ( `"npx cocalc-project --daemon" from "${ join ( root , "packages/project" ) }"` ) ; await promisify ( ( cb ) => { const child = spawn ( "npx" , [ "cocalc-project" , "--daemon" ] , { env , cwd : join ( root , "packages/project" ) , uid , gid : uid , } ) ; child . on ( "error" , ( err ) => { cb ( err ) ; } ) ; child . on ( "exit" , ( code ) => { cb ( code ) ; } ) ; } ) ( ) ; } async function exec ( command , verbose ? ) { winston . debug ( `exec '${ command }'` ) ; const output = await promisify ( exec0 ) ( command ) ; if ( verbose ) { winston . debug ( `output: ${ JSON . stringify ( output ) }` ) ; } return output ; } export async function createUser ( project_id ) { const username = getUsername ( project_id ) ; try { await exec ( `/usr/sbin/userdel ${ username }` ) ; } catch ( _ ) { } const uid = `${ getUid ( project_id ) }` ; winston . debug ( "createUser: adding group" ) ; await exec ( `/usr/sbin/groupadd -g ${ uid } -o ${ username }` , true ) ; winston . debug ( "createUser: adding user" ) ; await exec ( `/usr/sbin/useradd -u ${ uid } -g ${ uid } -o ${ username } -m -d ${ homePath ( project_id ) } -s /bin/bash` , true ) ; } export async function deleteUser ( project_id ) { const username = getUsername ( project_id ) ; const uid = `${ getUid ( project_id ) }` ; await exec ( `pkill -9 -u ${ uid } | true` ) ; try { await exec ( `/usr/sbin/userdel ${ username }` ) ; } catch ( _ ) { } } export function sanitizedEnv ( env ) { const env2 = { ... env } ; for ( const key of [ "PGDATA" , "PGHOST" , "NODE_ENV" , "NODE_OPTIONS" , "BASE_PATH" , "PORT" , "DATA" , "LOGS" , ] ) { delete env2 [ key ] ; } for ( const key in env2 ) { if ( key . startsWith ( "COCALC_" ) || env2 [ key ] == null ) { delete env2 [ key ] ; } } return env2 as { [ key ] : string ; } ; } export async function getEnvironment ( project_id ) { const extra = await callback2 ( db ( ) . get_project_extra_env , { project_id } ) ; const extra_env = Buffer . from ( JSON . stringify ( extra ?? { } ) ) . toString ( "base64" ) ; const USER = getUsername ( project_id ) ; const HOME = homePath ( project_id ) ; const DATA = dataPath ( HOME ) ; return { ... sanitizedEnv ( process . env ) , ... { HOME , BASE_PATH : base_path , DATA , LOGS : join ( DATA , "logs" ) , COCALC_PROJECT_ID : project_id , COCALC_USERNAME : USER , USER , COCALC_EXTRA_ENV : extra_env , PATH : `${ HOME }/bin:${ HOME }/.local/bin:${ process . env . PATH }` , } , } ; } export async function getState ( HOME ) { winston . debug ( `getState("${ HOME }")` ) ; try { return { ip : "localhost" , state : ( await isProjectRunning ( HOME ) ) ? "running" : "opened" , time : new Date ( ) , } ; } catch ( err ) { return { error : `${ err }` , time : new Date ( ) , state : "opened" , } ; } } export async function getStatus ( HOME ) { winston . debug ( `getStatus("${ HOME }")` ) ; const data = dataPath ( HOME ) ; const status = { } ; if ( ! ( await isProjectRunning ( HOME ) ) ) { return status ; } for ( const path of [ "project.pid" , "hub-server.port" , "browser-server.port" , "sage_server.port" , "sage_server.pid" , "secret_token" , ] ) { try { const val = ( await readFile ( join ( data , path ) ) ) . toString ( ) . trim ( ) ; if ( path . endsWith ( ".pid" ) ) { const pid = parseInt ( val ) ; if ( pidIsRunning ( pid ) ) { status [ path ] = pid ; } } else if ( path . endsWith ( ".port" ) ) { status [ path ] = parseInt ( val ) ; } else { status [ path ] = val ; } } catch ( _err ) { } } return status ; } export async function ensureConfFilesExists ( HOME , uid ? ) { for ( const path of [ "bashrc" , "bash_profile" ] ) { const target = join ( HOME , `.${ path }` ) ; try { await stat ( target ) ; } catch ( _ ) { const source = join ( root , "smc_pyutil/smc_pyutil/templates" , process . platform , path ) ; try { await copyFile ( source , target ) ; if ( uid != null ) { await chown ( target , uid ) ; } } catch ( err ) { winston . error ( `ensureConfFilesExists -- ${ err }` ) ; } } } } export async function copyPath ( opts , project_id , target_uid ? ) { winston . info ( `copyPath(source="${ project_id }"): opts=${ JSON . stringify ( opts ) }` ) ; const { path , overwrite_newer , delete_missing , backup , timeout , bwlimit } = opts ; if ( path == null ) { throw Error ( "path must be specified" ) ; } const target_project_id = opts . target_project_id ?? project_id ; const target_path = opts . target_path ?? path ; if ( ! is_valid_uuid_string ( project_id ) ) { throw Error ( `project_id=${ project_id } is invalid` ) ; } if ( ! is_valid_uuid_string ( target_project_id ) ) { throw Error ( `target_project_id=${ target_project_id } is invalid` ) ; } const sourceHome = homePath ( project_id ) ; const source_abspath = resolve ( join ( sourceHome , path ) ) ; if ( ! source_abspath . startsWith ( sourceHome ) ) { throw Error ( `source path must be contained in project home dir` ) ; } const targetHome = homePath ( target_project_id ) ; const target_abspath = resolve ( join ( targetHome , target_path ) ) ; if ( ! target_abspath . startsWith ( targetHome ) ) { throw Error ( `target path must be contained in target project home dir` ) ; } if ( source_abspath == target_abspath ) { return ; } const stats = await stat ( source_abspath ) ; const isDir = stats . isDirectory ( ) ; const args = [ ] ; if ( process . platform == "darwin" ) { args . push ( "-zax" ) ; } else { args . push ( ... [ "-zaxs" , "--omit-link-times" ] ) ; } if ( opts . exclude ) { for ( const pattern of opts . exclude ) { args . push ( "--exclude" ) ; args . push ( pattern ) ; } } if ( ! overwrite_newer ) { args . push ( "--update" ) ; } if ( backup ) { args . push ( "--backup" ) ; } if ( delete_missing ) { args . push ( "--delete" ) ; } if ( bwlimit ) { args . push ( `--bwlimit=${ bwlimit }` ) ; } if ( timeout ) { args . push ( `--timeout=${ timeout }` ) ; } if ( target_uid && target_project_id != project_id ) { args . push ( `--chown=${ target_uid }:${ target_uid }` ) ; } args . push ( source_abspath + ( isDir ? "/" : "" ) ) ; args . push ( target_abspath + ( isDir ? "/" : "" ) ) ; async function make_target_path ( ) { if ( isDir ) { await spawnAsync ( "mkdir" , [ "-p" , target_abspath ] , { uid : target_uid , gid : target_uid , } ) ; } else { await spawnAsync ( "mkdir" , [ "-p" , dirname ( target_abspath ) ] , { uid : target_uid , gid : target_uid , } ) ; } } try { await make_target_path ( ) ; } catch ( _err ) { await createUser ( target_project_id ) ; await make_target_path ( ) ; await deleteUser ( target_project_id ) ; } winston . info ( `doing rsync ${ args . join ( " " ) }` ) ; if ( opts . wait_until_done ?? true ) { try { const stdout = await spawnAsync ( "rsync" , args , { timeout : opts . timeout ? 1000 * opts . timeout : undefined , } ) ; winston . info ( `finished rsync ${ stdout }` ) ; } catch ( err ) { throw Error ( `WARNING: copy exited with an error -- ${ err . stderr } -- "rsync ${ args . join ( " " ) }"` ) ; } } else { spawn ( "rsync" , args , { timeout : opts . timeout } ) ; } }