const DEBUG_BREAK_AFTER_DIAGNOSTICS_COMPLETE = false ; import * as assert from "assert" ; import * as fse from "fs-extra" ; import * as path from 'path' ; import { Diagnostic , DiagnosticSeverity , Disposable , languages , Position , Range , TextDocument , Uri } from "vscode" ; import { parseError } from "vscode-azureextensionui" ; import { backendValidationDiagnosticsSource , diagnosticsCompletePrefix , expressionsDiagnosticsSource , ExpressionType , ext , LanguageServerState , languageServerStateSource , readUtf8FileWithBom } from "../../extension.bundle" ; import { bail } from "../global.test" ; import { DEFAULT_TESTCASE_TIMEOUT_MS , DISABLE_LANGUAGE_SERVER } from "../testConstants" ; import { delay } from "./delay" ; import { mapParameterFile } from "./mapParameterFile" ; import { parseParametersWithMarkers } from "./parseTemplate" ; import { positionToString , rangeToString } from "./rangeToString" ; import { resolveInTestFolder } from "./resolveInTestFolder" ; import { stringify } from "./stringify" ; import { TempDocument , TempEditor , TempFile } from "./TempFile" ; import { writeToLog } from "./testLog" ; export const defaultDiagnosticsTimeoutMs = DEFAULT_TESTCASE_TIMEOUT_MS - 2000 ; export type TransformDiagnosticMessage = ( msg : string ) => string ; export interface ICompareDiagnosticsOptions { transformResults ? : TransformDiagnosticMessage ; } enum ExpectedDiagnosticSeverity { Warning = 4 , Error = 8 } enum IncludeRange { no , range , position } export interface IExpectedDiagnostic { message : string ; severity : ExpectedDiagnosticSeverity ; source : string ; startLineNumber : number ; startColumn : number ; endLineNumber : number ; endColumn : number ; } export type ExpectedDiagnostics = string [ ] | IExpectedDiagnostic [ ] ; export interface DiagnosticSource { name : string ; } export const diagnosticSources = { expressions : { name : expressionsDiagnosticsSource } , schema : { name : 'arm-template (schema)' } , syntax : { name : 'arm-template (syntax)' } , backendValidation : { name : backendValidationDiagnosticsSource } , } ; function isSourceFromLanguageServer ( source ) { return source . name !== diagnosticSources . expressions . name ; } export interface IDeploymentParameterDefinition { type : ExpressionType | string ; metadata ? : { [ key ] : string | undefined ; description ? : string ; } ; maxLength ? : number ; minLength ? : number ; maxValue ? : number ; minValue ? : number ; defaultValue ? : number | unknown [ ] | string | { } ; allowedValues ? : ( number | unknown [ ] | string | { } ) [ ] ; } export interface IDeploymentOutput { type : ExpressionType | string ; value ? : number | unknown [ ] | string | { } ; copy ? : { count : string ; input : string ; } ; } export interface IDeploymentFunctionDefinition { parameters ? : { name : string ; type : ExpressionType | string ; } [ ] ; output ? : IDeploymentOutput ; } export interface IDeploymentNamespaceDefinition { namespace : string ; members : { [ key ] : IDeploymentFunctionDefinition ; } ; } export interface IDeploymentParameterValue { value : unknown ; } export interface IDeploymentParametersFile { "$schema" : "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#" | "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#" ; contentVersion : string ; parameters ? : { [ key ] : IDeploymentParameterValue ; } ; } export interface IDeploymentTemplate { "$schema" : "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#" | "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#" | string ; contentVersion : string ; apiProfile ? : string ; parameters ? : { [ key ] : IDeploymentParameterDefinition ; } ; variables ? : { copy ? : { name : string ; count : number ; input : string | { } ; } [ ] ; [ key ] : number | unknown [ ] | string | { } | undefined ; } ; resources : IDeploymentTemplateResource [ ] ; outputs ? : { [ key ] : IDeploymentOutput ; } ; functions ? : IDeploymentNamespaceDefinition [ ] ; } export interface IPartialDeploymentTemplate { "$schema" ? : "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#" | "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#" | string ; contentVersion ? : string ; apiProfile ? : string ; parameters ? : { [ key ] : Partial < IDeploymentParameterDefinition > ; } ; variables ? : { copy ? : { name : string ; count : number ; input : string | { } ; } [ ] ; [ key ] : number | unknown [ ] | string | { } | undefined ; } ; resources ? : IPartialDeploymentTemplateResource [ ] ; outputs ? : { [ key ] : Partial < IDeploymentOutput > ; } ; functions ? : Partial < IDeploymentNamespaceDefinition > [ ] ; } export const defaultArmSchema = "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#" ; export const minimalDeploymentTemplate = { "$schema" : defaultArmSchema , "contentVersion" : "1.0.0.0" , "resources" : [ ] } ; export interface IDeploymentTemplateResource { type : string ; name : string ; apiVersion ? : string ; location ? : string ; dependsOn ? : string [ ] ; tags ? : { [ key ] : string ; } | string ; properties ? : { [ key ] : unknown ; } ; resources ? : IDeploymentTemplateChildResource [ ] ; [ key ] : unknown ; } export interface IDeploymentTemplateChildResource { type : string ; name : string ; apiVersion : string ; location ? : string ; dependsOn ? : string [ ] ; tags ? : { [ key ] : string ; } | string ; properties ? : { [ key ] : unknown ; } ; resources ? : IDeploymentTemplateResource [ ] ; [ key ] : unknown ; } export interface IPartialDeploymentTemplateResource { type ? : string ; name ? : string ; apiVersion ? : string ; location ? : string ; dependsOn ? : string [ ] ; tags ? : { [ key ] : string ; } | string ; properties ? : { [ key ] : unknown ; } ; resources ? : IPartialDeploymentTemplateResource [ ] ; [ key ] : unknown ; } export type ITestDiagnosticsOptions = IGetDiagnosticsOptions & ICompareDiagnosticsOptions ; export interface IGetDiagnosticsOptions { /**
     * Timeout in milliseconds
     */ timeoutMs ? : number ; /**
     * Parameters that will be placed into a temp file and associated with the template file
     */ parameters ? : string | Partial < IDeploymentParametersFile > ; /**
     * A parameter file that will be associated with the template file
     */ parametersFile ? : string ; /**
     * Error sources to include in the comparison - defaults to all
     */ includeSources ? : DiagnosticSource [ ] ; /**
     * Error sources to ignore in the comparison - defaults to ignoring none
     */ ignoreSources ? : DiagnosticSource [ ] ; /**
     * Ignore informational diagnostics
     */ ignoreInfos ? : boolean ; /**
     * defaults to false - whether to include the error range in the results for comparison (if true, ignored when expected messages don't have ranges)
     */ includeRange ? : boolean ; /**
     * defaults to false - whether to include the error start position in the results for comparison (if true, ignored when expected messages don't have start positions)
     */ includePositions ? : boolean ; /**
     * Run a replacement using this regex and replacement on the file/contents before testing for errors
     */ search ? : RegExp ; /**
     * Run a replacement using this regex and replacement on the file/contents before testing for errors
     */ replace ? : string ; /**
     * Don't add schema (testDiagnostics only) automatically
     */ doNotAddSchema ? : boolean ; /**
     * Wait until the diagnostics version of all sources changes before retrieving themed.  Without this, it just waits until all diagnostics indicate they're complete.
     * NOTE: If previousResults are passed in, waitForChange is assumed true
     */ waitForChange ? : boolean ; /**
     * If specified, wait for the filter to return true as well as the normal tests before considering the diagnostics complete
     */ waitForDiagnosticsFilter ? ( results ) : boolean | Promise < boolean > ; } export async function testDiagnosticsFromFile ( filePath , options , expected ) { await testDiagnosticsCore ( async ( ) => await getDiagnosticsForTemplate ( filePath , 1 , options ) , options , expected ) ; } export async function testDiagnostics ( templateContentsOrFileName , options , expected ) { await testDiagnosticsCore ( async ( ) => await getDiagnosticsForTemplate ( templateContentsOrFileName , 1 , options ) , options , expected ) ; } async function testDiagnosticsCore ( getDiagnostics , options , expected ) { let actual = await getDiagnostics ( ) ; compareDiagnostics ( actual . diagnostics , expected , options ) ; } export async function testDiagnosticsFromUri ( documentUri , options , expected ) { await testDiagnosticsCore ( async ( ) => await getDiagnosticsForDocument ( documentUri , 1 , options ) , options , expected ) ; } export interface IDiagnosticsResults { diagnostics : Diagnostic [ ] ; sourceCompletionVersions : { [ source ] : number ; } ; } export async function getDiagnosticsForDocument ( document , expectedMinimumVersionForEachSource , options , previousResults ? ) { const documentUri = document instanceof Uri ? document : document . uri ; let dispose ; let timer ; let filterSources = Array . from ( Object . values ( diagnosticSources ) ) ; if ( options . includeSources ) { filterSources = filterSources . filter ( s => options . includeSources ! . find ( s2 => s2 . name === s . name ) ) ; } if ( options . ignoreSources ) { filterSources = filterSources . filter ( s => ! ( options . ignoreSources ! ) . includes ( s ) ) ; } const includesLanguageServerSource = filterSources . some ( isSourceFromLanguageServer ) ; if ( includesLanguageServerSource && DISABLE_LANGUAGE_SERVER ) { throw new Error ( "DISABLE_LANGUAGE_SERVER is set, but this test is trying to include a non-expressions diagnostic source" ) ; } const timeoutMs = options . timeoutMs ?? defaultDiagnosticsTimeoutMs ; let diagnosticsPromise = new Promise < IDiagnosticsResults > ( async ( resolve , reject ) : Promise < void > => { try { let currentDiagnostics ; function getCurrentDiagnostics ( ) { const sourceCompletionVersions = { } ; currentDiagnostics = languages . getDiagnostics ( documentUri ) ; currentDiagnostics = currentDiagnostics . filter ( d => d . source !== languageServerStateSource ) ; let filteredDiagnostics = currentDiagnostics . filter ( d => filterSources . find ( s => d . source === s . name ) ) ; if ( options . ignoreInfos ) { filteredDiagnostics = filteredDiagnostics . filter ( d => d . message . startsWith ( diagnosticsCompletePrefix ) || d . severity !== DiagnosticSeverity . Information ) ; } for ( let d of filteredDiagnostics ) { if ( d . message . startsWith ( diagnosticsCompletePrefix ) ) { const version = Number ( d . message . match ( /version ([0-9]+)/ ) ! [ 1 ] ) ; sourceCompletionVersions [ d . source ! ] = version ; } } filteredDiagnostics = filteredDiagnostics . filter ( d => ! d . message . startsWith ( diagnosticsCompletePrefix ) ) ; if ( includesLanguageServerSource ) { if ( ext . languageServerState === LanguageServerState . Failed ) { throw new Error ( `Language server failed on start-up: ${ ext . languageServerStartupError }` ) ; } else if ( ext . languageServerState === LanguageServerState . Stopped ) { throw new Error ( `Language server is in stopped state` ) ; } } return { diagnostics : filteredDiagnostics , sourceCompletionVersions } ; } function areAllSourcesComplete ( sourceCompletionVersions ) { for ( let source of filterSources ) { const completionVersion = sourceCompletionVersions [ source . name ] ; if ( completionVersion === undefined ) { return false ; } if ( requiredSourceCompletionVersions [ source . name ] !== undefined && completionVersion < requiredSourceCompletionVersions [ source . name ] ) { return false ; } } return true ; } async function diagnosticsFilterPasses ( results ) { if ( options . waitForDiagnosticsFilter ) { const filterValue = await options . waitForDiagnosticsFilter ( results ) ; return filterValue ; } return true ; } let currentResults = previousResults ?? getCurrentDiagnostics ( ) ; const requiredSourceCompletionVersions = Object . assign ( { } , currentResults . sourceCompletionVersions ) ; if ( options . waitForChange || previousResults ) { for ( let source in requiredSourceCompletionVersions ) { requiredSourceCompletionVersions [ source ] = requiredSourceCompletionVersions [ source ] + 1 ; } } if ( areAllSourcesComplete ( currentResults . sourceCompletionVersions ) && await diagnosticsFilterPasses ( currentResults ) ) { resolve ( currentResults ) ; return ; } writeToLog ( "Waiting for diagnostics to complete..." ) ; let done = false ; timer = setTimeout ( ( ) => { bail ( ) ; reject ( new Error ( 'Timed out waiting for diagnostics. Last retrieved diagnostics: ' + ( currentDiagnostics ? currentDiagnostics . map ( d => d . message ) . join ( '\n' ) : "None" ) ) ) ; done = true ; } , timeoutMs ) ; while ( ! done ) { const results = getCurrentDiagnostics ( ) ; if ( areAllSourcesComplete ( results . sourceCompletionVersions ) && await diagnosticsFilterPasses ( results ) ) { resolve ( results ) ; done = true ; } await delay ( 100 ) ; } } catch ( err ) { reject ( err ) ; } } ) ; let diagnostics = await diagnosticsPromise ; if ( DEBUG_BREAK_AFTER_DIAGNOSTICS_COMPLETE ) { debugger ; } if ( dispose ) { dispose . dispose ( ) ; } if ( timer ) { clearTimeout ( timer ) ; } writeToLog ( `Diagnostics complete:  ${ stringify ( diagnostics ) }` ) ; for ( const source of Object . getOwnPropertyNames ( diagnostics . sourceCompletionVersions ) ) { if ( ! ( diagnostics . sourceCompletionVersions [ source ] >= expectedMinimumVersionForEachSource ) ) { assert . fail ( `Expected diagnostics source to be at least 3, but found ${ stringify ( diagnostics ) }` ) ; } } return diagnostics ; } export async function getDiagnosticsForTemplate ( templateContentsOrFileName , expectedMinimumVersionForEachSource , options ? ) { let templateContents ; let templateFile ; let paramsFile ; let editor ; try { options = options || { } ; if ( typeof templateContentsOrFileName === 'string' ) { if ( ! ! templateContentsOrFileName . match ( /\.jsonc?$/ ) ) { let sourcePath = resolveInTestFolder ( templateContentsOrFileName ) ; templateFile = TempFile . fromExistingFile ( sourcePath ) ; } else { templateContents = templateContentsOrFileName ; } } else { const templateObject = templateContentsOrFileName ; templateContents = stringify ( templateObject ) ; } if ( options . search && ! templateContents && templateFile ) { templateContents = await readUtf8FileWithBom ( templateFile . fsPath ) ; templateFile = undefined ; } if ( ! templateFile ) { assert ( templateContents ) ; if ( ! options . doNotAddSchema && ! templateContents . includes ( '$schema' ) ) { templateContents = templateContents . replace ( /\s*{\s*/ , '{\n"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",\n' ) ; } if ( options . search ) { let newContents = templateContents . replace ( options . search , options . replace ! ) ; templateContents = newContents ; } templateFile = TempFile . fromContents ( templateContents , '.json' ) ; } const document = new TempDocument ( templateFile ) ; if ( options . parameters || options . parametersFile ) { if ( options . parameters ) { const { unmarkedText : unmarkedParams } = parseParametersWithMarkers ( options . parameters ) ; paramsFile = TempFile . fromContents ( unmarkedParams ) ; } else { assert ( options . parametersFile ) ; let absPath = path . join ( path . dirname ( templateFile . fsPath ) , options . parametersFile ) ; if ( ! fse . pathExistsSync ( absPath ) ) { absPath = resolveInTestFolder ( options . parametersFile ! ) ; } assert ( fse . pathExistsSync ( absPath ) , `Couldn't find parameter file ${ absPath }` ) ; paramsFile = TempFile . fromExistingFile ( absPath ) ; } await mapParameterFile ( templateFile . uri , paramsFile . uri ) ; } editor = new TempEditor ( document ) ; await editor . open ( ) ; let diagnostics = await getDiagnosticsForDocument ( document . realDocument , expectedMinimumVersionForEachSource , options ) ; assert ( diagnostics ) ; await editor . dispose ( ) ; return diagnostics ; } finally { if ( editor ) { await editor . dispose ( ) ; } if ( templateFile ) { await mapParameterFile ( templateFile . uri , undefined , false ) ; templateFile . dispose ( ) ; } if ( paramsFile ) { paramsFile . dispose ( ) ; } } } export function expectedDiagnosticToString ( diagnostic ) { return diagnosticToString ( { message : diagnostic . message , range : new Range ( new Position ( diagnostic . startLineNumber , diagnostic . startColumn ) , new Position ( diagnostic . endLineNumber , diagnostic . endColumn ) ) , severity : diagnostic . severity === ExpectedDiagnosticSeverity . Error ? DiagnosticSeverity . Error : diagnostic . severity === ExpectedDiagnosticSeverity . Warning ? DiagnosticSeverity . Warning : < DiagnosticSeverity > - 1 , source : diagnostic . source , code : "" } , { } , IncludeRange . range ) ; } export function diagnosticToString ( diagnostic , options , includeRange ) { assert ( diagnostic . code === '' , `Expecting empty code for all diagnostics, instead found Code="${ String ( diagnostic . code ) }" for "${ diagnostic . message }"` ) ; let severity = "" ; switch ( diagnostic . severity ) { case DiagnosticSeverity . Error : severity = "Error" ; break ; case DiagnosticSeverity . Warning : severity = "Warning" ; break ; case DiagnosticSeverity . Information : severity = "Information" ; break ; case DiagnosticSeverity . Hint : severity = "Hint" ; break ; default : assert . fail ( `Expected severity ${ diagnostic . severity }` ) ; } let s = `${ severity }: ${ diagnostic . message } (${ diagnostic . source })${ maybeRange ( diagnostic . range ) }` ; if ( diagnostic . relatedInformation ) { const related = diagnostic . relatedInformation [ 0 ] ; s = `${ s } [${ related . message }]${ maybeRange ( related . location . range ) }` ; } return s ; function maybeRange ( range ) { if ( includeRange === IncludeRange . range ) { return ' ' + rangeToString ( range ) ; } else if ( includeRange === IncludeRange . position ) { return ' ' + positionToString ( range ?. start ) ; } else { return "" ; } } } function compareDiagnostics ( actual , expected , options ) { transformResults ( actual , options ) ; let expectedHasRanges = expected . length === 0 || ( typeof expected [ 0 ] === 'string' && ! ! expected [ 0 ] . match ( /\[[0-9]+,[0-9]+-[0-9]+,[0-9]+\]/ ) || ( typeof expected [ 0 ] !== 'string' && expected [ 0 ] . startLineNumber !== undefined ) ) ; let expectedHasPositions = expected . length > 0 && ( typeof expected [ 0 ] === 'string' && ! ! expected [ 0 ] . match ( /\[[0-9]+,[0-9]+\]/ ) ) ; let includeRange = ( ! ! options . includeRange || expectedHasRanges ) ? IncludeRange . range : ( ! ! options . includePositions || expectedHasPositions ) ? IncludeRange . position : IncludeRange . no ; let expectedAsStrings = expected . length === 0 ? [ ] : typeof expected [ 0 ] === 'string' ? expected : ( < IExpectedDiagnostic [ ] > expected ) . map ( d => typeof d === 'string' ? d : expectedDiagnosticToString ( d ) ) ; let actualAsStrings = actual . map ( d => diagnosticToString ( d , options , includeRange ) ) ; expectedAsStrings = expectedAsStrings . sort ( ) ; actualAsStrings = actualAsStrings . sort ( ) ; try { assert . deepStrictEqual ( actualAsStrings , expectedAsStrings ) ; } catch ( err ) { const msgMax = 5000 ; let msg = parseError ( err ) . message ; if ( msg . length > msgMax ) { msg = `${ msg . slice ( 0 , msgMax ) }\n...(more)` ; } throw new Error ( msg ) ; } } function transformResults ( results , options ) { if ( options . transformResults ) { for ( const result of results ) { result . message = options . transformResults ( result . message ) ; } } } export function simplifyBadTypeResourceMessage ( msg ) { return msg . replace ( /Value must be one of the following values: [a-zA-Z0-9-, "./]+/ , "Value must be one of the following values: {...}" ) ; }