/**
 * Copyright 2021 Opstrace, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */ import fs from "fs" ; import { strict as assert } from "assert" ; import { ArgumentParser } from "argparse" ; import { ZonedDateTime } from "@js-joda/core" ; import { rndstring } from "./util" ; import { log , changeStderrLogLevel } from "./log" ; import { DEFAULT_LOG_LEVEL_STDERR } from "./index" ; interface CfgInterface { n_series : number ; n_chars_per_msg : number ; n_samples_per_series_fragment : number ; n_cycles : number ; n_fragments_per_push_message : number ; cycle_stop_write_after_n_fragments : number ; cycle_stop_write_after_n_seconds : number ; cycle_stop_write_after_n_seconds_jitter : number ; max_concurrent_writes : number ; max_concurrent_reads : number ; logLevel : string ; apibaseurl : string ; invocation_id : string ; log_start_time : string ; log_time_increment_ns : number ; http_server_port : number ; additional_labels : Array < [ string , string ] > | undefined ; compressability : string ; change_series_every_n_cycles : number ; metrics_mode : boolean ; metrics_time_increment_ms : number ; bearer_token_file : string ; retry_post_deadline_seconds : number ; retry_post_min_delay_seconds : number ; retry_post_max_delay_seconds : number ; retry_post_jitter : number ; skip_read : boolean ; read_n_series_only : number ; } export let CFG ; export let BEARER_TOKEN ; export function parseCmdlineArgs ( ) { const parser = new ArgumentParser ( { description : `Looker is a Loki / Cortex testing and benchmarking tool.

    Looker originated as a black-box storage testing program for Loki with
    strict and deep read validation. Since then, it has evolved quite a bit.

    ` } ) ; parser . add_argument ( "apibaseurl" , { help : "Push API base URL (/loki/api/v1/push and " + "/api/v1/push are prepended automatically in logs and metrics mode, " + "respectively)." , type : "str" } ) ; parser . add_argument ( "--log-level" , { help : `Set log level for output on stderr. ` + `One of: debug, info, warning, error. ` + `Default: ${ DEFAULT_LOG_LEVEL_STDERR }` , type : "str" , choices : [ "debug" , "info" , "warning" , "error" ] , default : DEFAULT_LOG_LEVEL_STDERR , metavar : "LEVEL" , dest : "logLevel" } ) ; parser . add_argument ( "--metrics-mode" , { help : "'Metrics mode' (Cortex) instead of 'Logs mode' (Loki)." , action : "store_true" , default : false } ) ; parser . add_argument ( "--n-series" , { help : "Number of synthetic time series to generate and consume from in each " + "write/read cycle." , type : "int" , metavar : "N" , required : true } ) ; parser . add_argument ( "--n-samples-per-series-fragment" , { help : "Number of samples (log entries or metric samples) to put into " + "each time series fragment. A time series fragment is a distinct set " + "of samples in chronological order. A push message is comprised of " + "one or more time series fragments (with F fragments from F different " + "time series -- a push message generated by Looker never contains " + "more than one time series fragment from the same time series." , type : "int" , metavar : "N" , required : true } ) ; parser . add_argument ( "--n-fragments-per-push-message" , { help : "Number of series fragments to serialize into a single binary push message " + "(HTTP POST request body), mixed from different series. Default: 1." , type : "int" , metavar : "N" , default : 1 } ) ; parser . add_argument ( "--n-chars-per-msg" , { help : "Number of characters per log message (ignored in metrics mode). " + "Default: 100." , type : "int" , metavar : "N" , default : 100 } ) ; parser . add_argument ( "--log-start-time" , { help : "This disables the loose coupling of synthetic time to wall time " + "and defines the timestamp of the first sample for all synthetic " + "log time series. Does not apply in metrics mode. Is useful when " + "writing to Loki that is configured to allow ingesting log samples " + "far from the past or future. Must be provided in " + "RFC3339 notation. Note that across cycles the same " + "start time is used." , type : "str" , default : "" } ) ; parser . add_argument ( "--log-time-increment-ns" , { help : "Time difference in nanonseconds between adjacent log entries in a " + "log series (between log entry timestamps) (ignored in metrics mode). " + "Default: 1 ns." , type : "int" , metavar : "N" , default : 1 } ) ; parser . add_argument ( "--metrics-time-increment-ms" , { help : "Time difference in milliseconds between adjacent samples in a " + "time series. Default: 1 ms." , type : "int" , metavar : "N" , default : 1 } ) ; parser . add_argument ( "--max-concurrent-writes" , { help : "Maximum number of POST HTTP requests to perform concurrently. " + "Default: use 10 concurrent writers or less if --n-series is smaller." , type : "int" , metavar : "N" , default : 0 } ) ; const readgroup = parser . add_mutually_exclusive_group ( ) ; readgroup . add_argument ( "--max-concurrent-reads" , { help : "Maximum number of GET HTTP requests to perform concurrently during " + "the read/validation phase. " + "Default: use 10 concurrent writers or less if --n-series is smaller." , type : "int" , metavar : "N" , default : 0 } ) ; readgroup . add_argument ( "--skip-read" , { help : "Skip the readout in the write/read cycle, proceed to the next " + "cycle instead." , action : "store_true" , default : false } ) ; parser . add_argument ( "--read-n-series-only" , { help : "Maximum number of series to read (validate) in the read phase of a " + "write/read cycle. Use this if you want to read (validate) less data " + "than what was written. The subset of series is picked randomly at the " + "beginning of each read phase. Default: 0 " + "(read back everything that was written)." , type : "int" , metavar : "N" , default : 0 } ) ; parser . add_argument ( "--compressability" , { help : "Compressability characteristic of generated log messages " + "(ignored in metrics mode)." , type : "str" , choices : [ "min" , "max" , "medium" ] , default : "min" } ) ; parser . add_argument ( "--n-cycles" , { help : "Number of write/read cycles to perform before terminating the program. " + "Every cycle allows for (potentially sparse) read validation of " + "what was previously written, and also generates a performance report. " + "Default: 0 (perform an infinite amount of cycles)." , type : "int" , metavar : "N" , default : 0 } ) ; parser . add_argument ( "--change-series-every-n-cycles" , { help : "Use the same log/metric time series for N cycles, then create a new " + "set of series (unique label sets). If set to 0 then the initial set " + "of time series is reused for the process lifetime (the default)." , type : "int" , metavar : "N" , default : 0 } ) ; parser . add_argument ( "--label" , { help : "add a label key/value pair to all emitted log entries" , metavar : [ "KEY" , "VALUE" ] , nargs : 2 , action : "append" , dest : "additional_labels" } ) ; parser . add_argument ( "--http-server-port" , { help : "HTTP server listen port (serves /metrics Prometheus endpoint). " + "Default: try 8900-8990." , type : "int" , default : 0 } ) ; const stopgroup = parser . add_mutually_exclusive_group ( { required : true } ) ; stopgroup . add_argument ( "--cycle-stop-write-after-n-fragments" , { help : "Cycle write phase stop criterion A: stop write (and enter read phase) " + "when  this many fragments were written for each time series. " + "Default: 0 (never enter the read phase)." , type : "int" , metavar : "N" , default : 0 } ) ; stopgroup . add_argument ( "--cycle-stop-write-after-n-seconds" , { help : "Cycle write phase stop criterion A: stop write (and enter read phase) " + "having written for approximately that many seconds. " + "Default: 0 (never enter the read phase)." , type : "int" , metavar : "N" , default : 0 } ) ; parser . add_argument ( "--cycle-stop-write-after-n-seconds-jitter" , { help : "Add random number of seconds from interval [-J,J] " + "to --cycle-stop-write-after-n-seconds. Default: 0." , metavar : "J" , type : "float" , default : 0 } ) ; parser . add_argument ( "--retry-post-deadline-seconds" , { help : "Maximum time spent retrying POST requests, in seconds. Default: 420." , type : "int" , default : 420 } ) ; parser . add_argument ( "--retry-post-min-delay-seconds" , { help : "Minimal delay between POST request retries, in seconds. Default: 3." , type : "int" , default : 3 } ) ; parser . add_argument ( "--retry-post-max-delay-seconds" , { help : "Maximum delay between POST request retries, in seconds. Deault: 30." , type : "int" , default : 30 } ) ; parser . add_argument ( "--retry-post-jitter" , { help : "Relative jitter to apply for calculating the retry delay " + "(1: max). Default: 0.5." , type : "float" , metavar : "FLOAT" , default : 0.5 } ) ; parser . add_argument ( "--bearer-token-file" , { help : "Read authentication token from file. Add header " + "`Authorization: Bearer <token>` to each HTTP request." , type : "str" , default : "" } ) ; parser . add_argument ( "--invocation-id" , { help : "Name of this looker instance to be included in labels. " + "By default a random looker-<X> string is used. " + "If read validation is enabled then care should be taken to ensure " + "uniqueness at the server." , type : "str" , default : "" } ) ; CFG = parser . parse_args ( ) ; changeStderrLogLevel ( CFG . logLevel ) ; if ( CFG . invocation_id === "" ) { const uniqueInvocationId = `looker-${ rndstring ( 10 ) }` ; CFG . invocation_id = uniqueInvocationId ; } if ( CFG . log_start_time !== "" ) { ZonedDateTime . parse ( CFG . log_start_time ) ; assert ( ! CFG . metrics_mode ) ; } if ( CFG . cycle_stop_write_after_n_seconds !== 0 && CFG . cycle_stop_write_after_n_fragments !== 0 ) { log . error ( "Only one per-cycle write stop criterion must be defined" ) ; process . exit ( 1 ) ; } if ( CFG . change_series_every_n_cycles > CFG . n_cycles ) { log . error ( "--change-series-every-n-cycles must not be larger than --n-cycles" ) ; process . exit ( 1 ) ; } if ( CFG . change_series_every_n_cycles < 0 ) { log . error ( "--change-series-every-n-cycles must not be negative" ) ; process . exit ( 1 ) ; } if ( CFG . max_concurrent_writes > CFG . n_series ) { log . error ( "--max-concurrent-writes must not be larger than --n-series" ) ; process . exit ( 1 ) ; } if ( CFG . cycle_stop_write_after_n_seconds_jitter ) { if ( CFG . cycle_stop_write_after_n_seconds === 0 ) { log . error ( "--cycle-stop-write-after-n-seconds-jitter can only be used with --cycle-stop-write-after-n-seconds" ) ; process . exit ( 1 ) ; } if ( CFG . cycle_stop_write_after_n_seconds_jitter > CFG . cycle_stop_write_after_n_seconds ) { log . error ( "--cycle-stop-write-after-n-seconds-jitter must be smaller than --cycle-stop-write-after-n-seconds" ) ; process . exit ( 1 ) ; } } if ( CFG . max_concurrent_writes === 0 ) { CFG . max_concurrent_writes = Math . min ( 10 , CFG . n_series ) ; } if ( CFG . n_fragments_per_push_message * CFG . max_concurrent_writes > CFG . n_series ) { log . error ( `--n-fragments-per-push-message (${ CFG . n_fragments_per_push_message }) ` + "multiplied with max_concurrent_writes " + `(${ CFG . max_concurrent_writes }) = ${ CFG . n_fragments_per_push_message * CFG . max_concurrent_writes } ` + `must not be larger than --n-series (${ CFG . n_series })` ) ; process . exit ( 1 ) ; } if ( CFG . max_concurrent_reads === 0 ) { CFG . max_concurrent_reads = Math . min ( 10 , CFG . n_series ) ; } if ( CFG . bearer_token_file !== "" ) { try { BEARER_TOKEN = fs . readFileSync ( CFG . bearer_token_file , { encoding : "utf8" } ) . trim ( ) ; } catch ( err ) { log . error ( "failed to read file %s: %s" , CFG . bearer_token_file , err . message ) ; process . exit ( 1 ) ; } log . info ( "authentication token read from --bearer-token-file `%s` (%s characters)" , CFG . bearer_token_file , BEARER_TOKEN . length ) ; } if ( CFG . retry_post_jitter <= 0 || CFG . retry_post_jitter >= 1 ) { log . error ( "--retry-post-jitter must be larger than 0 and smaller than 1" ) ; process . exit ( 1 ) ; } if ( CFG . skip_read && CFG . read_n_series_only !== 0 ) { log . error ( "--skip-read must not be used with --read-n-series-only" ) ; process . exit ( 1 ) ; } if ( CFG . read_n_series_only > CFG . n_series ) { log . error ( "--read-n-series-only must not be larger than --n-series" ) ; process . exit ( 1 ) ; } log . info ( "rendered config:\n%s" , JSON . stringify ( CFG , null , 2 ) ) ; }