import debug from "debug" ; const L = debug ( "project:project-info:server" ) ; import { delay } from "awaiting" ; import { join } from "path" ; import { exec } from "./utils" ; import { options } from "../init-program" ; import { promises as fsPromises } from "fs" ; import { pid2path as terminal_pid2path } from "../terminal/server" ; import { get_path_for_pid as x11_pid2path } from "../x11/server" ; import { get_kernel_by_pid } from "../jupyter/jupyter" ; const { readFile , readdir , readlink } = fsPromises ; import { check as df } from "diskusage" ; import { EventEmitter } from "events" ; import { Cpu , Process , Processes , ProjectInfo , Stat , State , DiskUsage , CoCalcInfo , CGroup , } from "./types" ; function is_in_dev_project ( ) { return process . env . SMC_LOCAL_HUB_HOME != null ; } const LIMIT = 100 ; const bytes2MiB = ( bytes ) => bytes / ( 1024 * 1024 ) ; export class ProjectInfoServer extends EventEmitter { private last ? : ProjectInfo = undefined ; private readonly dbg : Function ; private running = false ; private readonly testing : boolean ; private ticks : number ; private pagesize : number ; private delay_s : number ; constructor ( testing = false ) { super ( ) ; this . delay_s = 2 ; this . testing = testing ; this . dbg = L ; } public latest ( ) { return this . last ; } private async uptime ( ) { const out = await readFile ( "/proc/uptime" , "utf8" ) ; const uptime = parseFloat ( out . split ( " " ) [ 0 ] ) ; const boottime = new Date ( new Date ( ) . getTime ( ) - 1000 * uptime ) ; return [ uptime , boottime ] ; } private async stat ( path ) { const raw = await readFile ( path , "utf8" ) ; const [ i , j ] = [ raw . indexOf ( "(" ) , raw . lastIndexOf ( ")" ) ] ; const start = raw . slice ( 0 , i - 1 ) . trim ( ) ; const end = raw . slice ( j + 1 ) . trim ( ) ; const data = `${ start } comm ${ end }` . split ( " " ) ; const get = ( idx ) => parseInt ( data [ idx ] ) ; const ret = { ppid : get ( 3 ) , state : data [ 2 ] as State , utime : get ( 13 ) / this . ticks , stime : get ( 14 ) / this . ticks , cutime : get ( 15 ) / this . ticks , cstime : get ( 16 ) / this . ticks , starttime : get ( 21 ) / this . ticks , nice : get ( 18 ) , num_threads : get ( 19 ) , mem : { rss : ( get ( 23 ) * this . pagesize ) / ( 1024 * 1024 ) } , } ; return ret ; } private dt ( timestamp ) { return ( timestamp - ( this . last ?. timestamp ?? 0 ) ) / 1000 ; } private cpu ( { pid , stat , timestamp } ) { const total_cpu = stat . utime + stat . stime ; const prev_cpu = this . last ?. processes ?. [ pid ] ?. cpu . secs ?? total_cpu ; const dt = this . dt ( timestamp ) ; const pct = 100 * ( ( total_cpu - prev_cpu ) / dt ) ; return { pct : pct , secs : total_cpu } ; } private async cmdline ( path ) { return ( await readFile ( path , "utf8" ) ) . split ( "\0" ) . filter ( ( c ) => c . length > 0 ) ; } private cocalc ( { pid , cmdline } ) { if ( pid === process . pid ) { return { type : "project" } ; } const jupyter_kernel = get_kernel_by_pid ( pid ) ; if ( jupyter_kernel != null ) { return { type : "jupyter" , path : jupyter_kernel . get_path ( ) } ; } const termpath = terminal_pid2path ( pid ) ; if ( termpath != null ) { return { type : "terminal" , path : termpath } ; } const x11_path = x11_pid2path ( pid ) ; if ( x11_path != null ) { return { type : "x11" , path : x11_path } ; } if ( cmdline . length === 1 && cmdline [ 0 ] . startsWith ( "sshd:" ) && cmdline [ 0 ] . indexOf ( "-p 2222" ) != - 1 ) { return { type : "sshd" } ; } } private async process ( { pid : pid_str , uptime , timestamp } ) { const base = join ( "/proc" , pid_str ) ; const pid = parseInt ( pid_str ) ; const fn = ( name ) => join ( base , name ) ; const [ cmdline , exe , stat ] = await Promise . all ( [ this . cmdline ( fn ( "cmdline" ) ) , readlink ( fn ( "exe" ) ) , this . stat ( fn ( "stat" ) ) , ] ) ; const data = { pid , ppid : stat . ppid , cmdline , exe , stat , cpu : this . cpu ( { pid , timestamp , stat } ) , uptime : uptime - stat . starttime , cocalc : this . cocalc ( { pid , cmdline } ) , } ; return data ; } private async processes ( { timestamp , uptime } ) { const procs = { } ; let n = 0 ; for ( const pid of await readdir ( "/proc" ) ) { if ( ! pid . match ( /^[0-9]+$/ ) ) continue ; try { const proc = await this . process ( { pid , uptime , timestamp } ) ; procs [ proc . pid ] = proc ; } catch ( err ) { if ( this . testing ) this . dbg ( `process ${ pid } likely vanished – could happen – ${ err }` ) ; } if ( n > LIMIT ) { this . dbg ( `too many processes – limit of ${ LIMIT } reached!` ) ; break ; } else { n += 1 ; } } return procs ; } private async cgroup ( { timestamp } ) { if ( ! is_in_dev_project ( ) && ! options . kucalc && ! this . testing ) return ; const [ mem_stat_raw , cpu_raw , oom_raw , cfs_quota_raw , cfs_period_raw , ] = await Promise . all ( [ readFile ( "/sys/fs/cgroup/memory/memory.stat" , "utf8" ) , readFile ( "/sys/fs/cgroup/cpu,cpuacct/cpuacct.usage" , "utf8" ) , readFile ( "/sys/fs/cgroup/memory/memory.oom_control" , "utf8" ) , readFile ( "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us" , "utf8" ) , readFile ( "/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us" , "utf8" ) , ] ) ; const mem_stat_keys = [ "total_rss" , "total_cache" , "hierarchical_memory_limit" , ] ; const cpu_usage = parseFloat ( cpu_raw ) / Math . pow ( 10 , 9 ) ; const dt = this . dt ( timestamp ) ; const cpu_usage_rate = this . last ?. cgroup != null ? ( cpu_usage - this . last . cgroup . cpu_usage ) / dt : 0 ; const [ cfs_quota , cfs_period ] = [ parseInt ( cfs_quota_raw ) , parseInt ( cfs_period_raw ) , ] ; const mem_stat = mem_stat_raw . split ( "\n" ) . map ( ( line ) => line . split ( " " ) ) . filter ( ( [ k , _ ] ) => mem_stat_keys . includes ( k ) ) . reduce ( ( stat , [ key , val ] ) => { stat [ key ] = bytes2MiB ( parseInt ( val ) ) ; return stat ; } , { } ) ; const oom_kills = oom_raw . split ( "\n" ) . filter ( ( val ) => val . startsWith ( "oom_kill " ) ) . map ( ( val ) => parseInt ( val . slice ( "oom_kill " . length ) ) ) [ 0 ] ; return { mem_stat , cpu_usage , cpu_usage_rate , cpu_cores_limit : cfs_quota / cfs_period , oom_kills , } ; } private async disk_usage ( ) { const convert = function ( val ) { return { total : bytes2MiB ( val . total ) , free : bytes2MiB ( val . free ) , available : bytes2MiB ( val . available ) , usage : bytes2MiB ( val . total - val . free ) , } ; } ; const [ tmp , project ] = await Promise . all ( [ df ( "/tmp" ) , df ( process . env . HOME ?? "/home/user" ) , ] ) ; return { tmp : convert ( tmp ) , project : convert ( project ) } ; } private async init ( ) { if ( this . ticks == null ) { const [ p_ticks , p_pagesize ] = await Promise . all ( [ exec ( "getconf CLK_TCK" ) , exec ( "getconf PAGESIZE" ) , ] ) ; this . ticks = parseInt ( p_ticks . stdout . trim ( ) ) ; this . pagesize = parseInt ( p_pagesize . stdout . trim ( ) ) ; } } private async get_info ( ) { const [ uptime , boottime ] = await this . uptime ( ) ; const timestamp = new Date ( ) . getTime ( ) ; const [ processes , cgroup , disk_usage ] = await Promise . all ( [ this . processes ( { uptime , timestamp } ) , this . cgroup ( { timestamp } ) , this . disk_usage ( ) , ] ) ; const info = { timestamp , processes , uptime , boottime , cgroup , disk_usage , } ; return info ; } public stop ( ) { this . running = false ; } public async start ( ) { if ( this . running ) { this . dbg ( "project-info/server: already running, cannot be started twice" ) ; } else { await this . _start ( ) ; } } private async _start ( ) { this . dbg ( "start" ) ; if ( this . running ) { throw Error ( "Cannot start ProjectInfoServer twice" ) ; } this . running = true ; await this . init ( ) ; while ( true ) { const info = await this . get_info ( ) ; this . last = info ; this . emit ( "info" , info ) ; if ( this . running ) { await delay ( 1000 * this . delay_s ) ; } else { this . dbg ( "start: no longer running → stopping loop" ) ; this . last = undefined ; return ; } if ( this . last != null && this . testing ) { const info = await this . get_info ( ) ; this . dbg ( JSON . stringify ( info , null , 2 ) ) ; return ; } } } } if ( require . main === module ) { const pis = new ProjectInfoServer ( true ) ; pis . start ( ) . then ( ( ) => process . exit ( ) ) ; }