import Debug from "debug" ; import Bluebird from "bluebird" ; import Bigtable from "@google-cloud/bigtable" ; import { Yildiz } from "../Yildiz" ; import { CountObject } from "../../interfaces/TTL" ; const debug = Debug ( "yildiz:metadata" ) ; const MAIN_KEY = "metadata" ; const DEFAULT_SAVE_INTERVAL_SEC = 2 ; const DEFAULT_COUNTS_INTERVAL_SEC = 5 ; const DEFAULT_KEYS = [ "nodes" , "edges" , "ttls" , ] ; export class Metadata { private yildiz : Yildiz ; private counts : CountObject ; private tov ! : NodeJS . Timer | number ; private tovGetCount ! : NodeJS . Timer | number ; private metadataTable : Bigtable . Table ; private columnFamilyMetadata : Bigtable . Family ; private promiseConcurrency : number ; constructor ( yildiz ) { this . yildiz = yildiz ; this . counts = { edges : 0 , nodes : 0 , ttls : 0 , } ; const { metadataTable , columnFamilyMetadata , } = this . yildiz . models ; this . metadataTable = metadataTable ; this . columnFamilyMetadata = columnFamilyMetadata ; this . promiseConcurrency = this . yildiz . config . promiseConcurrency || 1000 ; } public async init ( ) { this . reset ( ) ; const rowExists = await this . metadataTable . row ( MAIN_KEY ) . exists ( ) ; const cfName = this . columnFamilyMetadata . id ; if ( ! rowExists || ! rowExists [ 0 ] ) { await this . metadataTable . row ( MAIN_KEY ) . create ( { entry : { [ cfName ] : { created : 1 , } , } , } ) ; } const { saveMetadataInSec = DEFAULT_SAVE_INTERVAL_SEC , readMetadataInSec = DEFAULT_COUNTS_INTERVAL_SEC , } = this . yildiz . config . metadata || { } ; debug ( `running interval to cache metadata every ${ saveMetadataInSec } seconds` ) ; this . runIntervalSave ( saveMetadataInSec ) ; debug ( `running interval to read metadata every ${ readMetadataInSec } seconds` ) ; this . runIntervalCounts ( readMetadataInSec ) ; } private reset ( ) { DEFAULT_KEYS . forEach ( ( key ) => { this . counts [ key ] = 0 ; } ) ; } public async getAllCount ( ) { const row = this . metadataTable . row ( MAIN_KEY ) ; const counts = await row . get ( ) ; const valueRaw = counts [ 0 ] . data [ MAIN_KEY ] ; const value = { edges : 0 , nodes : 0 , ttls : 0 , } ; DEFAULT_KEYS . forEach ( ( key ) => { value [ key ] = this . counts [ key ] || 0 ; if ( valueRaw [ key ] && valueRaw [ key ] [ 0 ] && valueRaw [ key ] [ 0 ] . value ) { value [ key ] += valueRaw [ key ] [ 0 ] . value ; } } ) ; return value ; } public async getCount ( key ) { let value = 0 ; const cfName = this . columnFamilyMetadata . id ; const row = this . metadataTable . row ( MAIN_KEY ) ; const countsRow = await row . get ( `${ cfName }:${ key }` ) ; if ( countsRow [ 0 ] && countsRow [ 0 ] . data && countsRow [ 0 ] . data [ MAIN_KEY ] && countsRow [ 0 ] . data [ MAIN_KEY ] [ key ] && countsRow [ 0 ] . data [ MAIN_KEY ] [ key ] [ 0 ] && countsRow [ 0 ] . data [ MAIN_KEY ] [ key ] [ 0 ] . value ) { value = countsRow [ 0 ] . data [ MAIN_KEY ] [ key ] [ 0 ] . value ; } value += ( this . counts [ key ] || 0 ) ; return value < 0 ? 0 : value ; } public increaseCount ( key , amount = 1 ) { if ( typeof this . counts [ key ] === undefined ) { return 0 ; } ( this . counts [ key ] ) += amount ; return this . counts [ key ] ; } public decreaseCount ( key , amount = 1 ) { if ( typeof this . counts [ key ] === undefined ) { return 0 ; } ( this . counts [ key ] ) -= amount ; return this . counts [ key ] ; } /**
     * save syncs the in memory state of this.counts
     * to the single metadata row as qualifier increments
     * these are atomic, in memory state will be reset to 0
     */ private async save ( ) { const cfname = this . columnFamilyMetadata . id ; const row = this . metadataTable . row ( MAIN_KEY ) ; return await Bluebird . map ( Object . keys ( this . counts ) , async ( key ) => { if ( this . counts [ key ] === 0 || isNaN ( this . counts [ key ] ) ) { return 0 ; } const val = this . counts [ key ] ; await row . increment ( `${ cfname }:${ key }` , val ) ; this . counts [ key ] = 0 ; return val ; } , { concurrency : this . promiseConcurrency , } ) ; } private runIntervalSave ( saveMetadataInSec ) { this . tov = setTimeout ( async ( ) => { try { await this . save ( ) ; } catch ( err ) { debug ( "saving to metadata failed" , err ) ; } this . runIntervalSave ( saveMetadataInSec ) ; } , saveMetadataInSec * 1000 ) ; } private runIntervalCounts ( intervalSec ) { this . tovGetCount = setTimeout ( async ( ) => { try { const counts = await this . getAllCount ( ) ; this . yildiz . metrics . set ( "nodes_count" , counts . nodes ) ; this . yildiz . metrics . set ( "edges_count" , counts . edges ) ; this . yildiz . metrics . set ( "ttls_count" , counts . ttls ) ; } catch ( err ) { debug ( "failed to get all count" , err ) ; } this . runIntervalCounts ( intervalSec ) ; } , intervalSec * 1000 ) ; } public async close ( ) { try { await this . save ( ) ; } catch ( error ) { debug ( error ) ; } if ( this . tov ) { clearTimeout ( this . tov as NodeJS . Timer ) ; } if ( this . tovGetCount ) { clearTimeout ( this . tovGetCount as NodeJS . Timer ) ; } } }