import Char from 'typescript-char' ; import { assert } from '../common/debug' ; import { Diagnostic , DiagnosticAddendum } from '../common/diagnostic' ; import { DiagnosticSink } from '../common/diagnosticSink' ; import { convertOffsetsToRange , convertPositionToOffset } from '../common/positionUtils' ; import { latestStablePythonVersion , PythonVersion } from '../common/pythonVersion' ; import { TextRange } from '../common/textRange' ; import { TextRangeCollection } from '../common/textRangeCollection' ; import { timingStats } from '../common/timing' ; import { Localizer } from '../localization/localize' ; import { ArgumentCategory , ArgumentNode , ArrowCallableNode , ArrowCallableParameter , AssertNode , AssignmentExpressionNode , AssignmentNode , AugmentedAssignmentNode , AwaitNode , BinaryOperationNode , BreakNode , CallNode , CaseNode , ClassNode , ConstantNode , ContinueNode , DecoratorNode , DelNode , DictionaryEntryNode , DictionaryExpandEntryNode , DictionaryKeyEntryNode , DictionaryNode , EllipsisNode , ErrorExpressionCategory , ErrorNode , ExceptNode , ExpressionNode , extendRange , FormatStringNode , ForNode , FunctionAnnotationNode , FunctionNode , getNextNodeId , GlobalNode , IfNode , ImportAsNode , ImportFromAsNode , ImportFromNode , ImportNode , IndexNode , LambdaNode , ListComprehensionForNode , ListComprehensionIfNode , ListComprehensionIterNode , ListComprehensionNode , ListNode , MatchNode , MemberAccessNode , ModuleNameNode , ModuleNode , NameNode , NonlocalNode , NumberNode , ParameterCategory , ParameterNode , ParseNode , ParseNodeType , PassNode , PatternAsNode , PatternAtomNode , PatternCaptureNode , PatternClassArgumentNode , PatternClassNode , PatternLiteralNode , PatternMappingEntryNode , PatternMappingExpandEntryNode , PatternMappingKeyEntryNode , PatternMappingNode , PatternSequenceNode , PatternValueNode , RaiseNode , ReturnNode , SetNode , SliceNode , StatementListNode , StatementNode , StringListNode , StringNode , SuiteNode , TernaryNode , TryNode , TupleNode , TypeAnnotationNode , UnaryOperationNode , UnpackNode , WhileNode , WithItemNode , WithNode , YieldFromNode , YieldNode , } from './parseNodes' ; import * as StringTokenUtils from './stringTokenUtils' ; import { Tokenizer , TokenizerOutput } from './tokenizer' ; import { DedentToken , IdentifierToken , IndentToken , KeywordToken , KeywordType , NumberToken , OperatorToken , OperatorType , StringToken , StringTokenFlags , Token , TokenType , } from './tokenizerTypes' ; interface ListResult < T > { list : T [ ] ; trailingComma : boolean ; parseError ? : ErrorNode | undefined ; } interface SubscriptListResult { list : ArgumentNode [ ] ; trailingComma : boolean ; } export class ParseOptions { constructor ( ) { this . isStubFile = false ; this . pythonVersion = latestStablePythonVersion ; this . reportInvalidStringEscapeSequence = false ; this . skipFunctionAndClassBody = false ; } isStubFile : boolean ; pythonVersion : PythonVersion ; reportInvalidStringEscapeSequence : boolean ; skipFunctionAndClassBody : boolean ; } export interface ParseResults { text : string ; parseTree : ModuleNode ; importedModules : ModuleImport [ ] ; futureImports : Map < string , boolean > ; tokenizerOutput : TokenizerOutput ; containsWildcardImport : boolean ; } export interface ParseExpressionTextResults { parseTree ? : ExpressionNode | undefined ; lines : TextRangeCollection < TextRange > ; diagnostics : Diagnostic [ ] ; } export interface ModuleImport { nameNode : ModuleNameNode ; leadingDots : number ; nameParts : string [ ] ; importedSymbols : string [ ] | undefined ; } const enum ParseTextMode { Expression , VariableAnnotation , FunctionAnnotation } const supportPEP637 = false ; export class Parser { private _fileContents ? : string ; private _tokenizerOutput ? : TokenizerOutput ; private _tokenIndex = 0 ; private _areErrorsSuppressed = false ; private _parseOptions : ParseOptions = new ParseOptions ( ) ; private _diagSink : DiagnosticSink = new DiagnosticSink ( ) ; private _isInLoop = false ; private _isInFunction = false ; private _isInFinally = false ; private _isParsingTypeAnnotation = false ; private _isParsingIndexTrailer = false ; private _futureImportMap = new Map < string , boolean > ( ) ; private _importedModules : ModuleImport [ ] = [ ] ; private _containsWildcardImport = false ; private _assignmentExpressionsAllowed = true ; private _typingImportAliases : string [ ] = [ ] ; private _typingSymbolAliases : Map < string , string > = new Map < string , string > ( ) ; parseSourceFile ( fileContents , parseOptions , diagSink ) { timingStats . tokenizeFileTime . timeOperation ( ( ) => { this . _startNewParse ( fileContents , 0 , fileContents . length , parseOptions , diagSink ) ; } ) ; const moduleNode = ModuleNode . create ( { start : 0 , length : fileContents . length } ) ; timingStats . parseFileTime . timeOperation ( ( ) => { while ( ! this . _atEof ( ) ) { if ( ! this . _consumeTokenIfType ( TokenType . NewLine ) ) { const nextToken = this . _peekToken ( ) ; if ( nextToken . type === TokenType . Indent ) { this . _getNextToken ( ) ; const indentToken = nextToken as IndentToken ; if ( indentToken . isIndentAmbiguous ) { this . _addError ( Localizer . Diagnostic . inconsistentTabs ( ) , indentToken ) ; } else { this . _addError ( Localizer . Diagnostic . unexpectedIndent ( ) , nextToken ) ; } } const statement = this . _parseStatement ( ) ; if ( ! statement ) { this . _consumeTokensUntilType ( [ TokenType . NewLine ] ) ; } else { statement . parent = moduleNode ; moduleNode . statements . push ( statement ) ; } } } } ) ; assert ( this . _tokenizerOutput !== undefined ) ; return { text : fileContents , parseTree : moduleNode , importedModules : this . _importedModules , futureImports : this . _futureImportMap , tokenizerOutput : this . _tokenizerOutput ! , containsWildcardImport : this . _containsWildcardImport , } ; } parseTextExpression ( fileContents , textOffset , textLength , parseOptions , parseTextMode = ParseTextMode . Expression , initialParenDepth = 0 ) { const diagSink = new DiagnosticSink ( ) ; this . _startNewParse ( fileContents , textOffset , textLength , parseOptions , diagSink , initialParenDepth ) ; let parseTree ; if ( parseTextMode === ParseTextMode . VariableAnnotation ) { parseTree = this . _parseTypeAnnotation ( ) ; } else if ( parseTextMode === ParseTextMode . FunctionAnnotation ) { parseTree = this . _parseFunctionTypeAnnotation ( ) ; } else { const exprListResult = this . _parseTestOrStarExpressionList ( false ) ; if ( exprListResult . parseError ) { parseTree = exprListResult . parseError ; } else { if ( exprListResult . list . length === 0 ) { this . _addError ( Localizer . Diagnostic . expectedExpr ( ) , this . _peekToken ( ) ) ; } parseTree = this . _makeExpressionOrTuple ( exprListResult , false ) ; } } if ( this . _peekTokenType ( ) === TokenType . NewLine ) { this . _getNextToken ( ) ; } if ( ! this . _atEof ( ) ) { this . _addError ( Localizer . Diagnostic . unexpectedExprToken ( ) , this . _peekToken ( ) ) ; } return { parseTree , lines : this . _tokenizerOutput ! . lines , diagnostics : diagSink . fetchAndClear ( ) , } ; } private _startNewParse ( fileContents , textOffset , textLength , parseOptions , diagSink , initialParenDepth = 0 ) { this . _fileContents = fileContents ; this . _parseOptions = parseOptions ; this . _diagSink = diagSink ; const tokenizer = new Tokenizer ( ) ; this . _tokenizerOutput = tokenizer . tokenize ( fileContents , textOffset , textLength , initialParenDepth ) ; this . _tokenIndex = 0 ; } private _parseStatement ( ) { if ( this . _consumeTokenIfType ( TokenType . Dedent ) ) { this . _addError ( Localizer . Diagnostic . unexpectedUnindent ( ) , this . _peekToken ( ) ) ; } switch ( this . _peekKeywordType ( ) ) { case KeywordType . If : return this . _parseIfStatement ( ) ; case KeywordType . While : return this . _parseWhileStatement ( ) ; case KeywordType . For : return this . _parseForStatement ( ) ; case KeywordType . Try : return this . _parseTryStatement ( ) ; case KeywordType . With : return this . _parseWithStatement ( ) ; case KeywordType . Def : return this . _parseFunctionDef ( ) ; case KeywordType . Class : return this . _parseClassDef ( ) ; case KeywordType . Async : return this . _parseAsyncStatement ( ) ; case KeywordType . Match : { const peekToken = this . _peekToken ( 1 ) ; let isInvalidMatchToken = false ; if ( peekToken . type === TokenType . Colon || peekToken . type === TokenType . Semicolon || peekToken . type === TokenType . Comma || peekToken . type === TokenType . Dot || peekToken . type === TokenType . NewLine || peekToken . type === TokenType . EndOfStream ) { isInvalidMatchToken = true ; } else if ( peekToken . type === TokenType . Operator ) { const operatorToken = peekToken as OperatorToken ; if ( operatorToken . operatorType !== OperatorType . Multiply && operatorToken . operatorType !== OperatorType . Subtract ) { isInvalidMatchToken = true ; } } if ( ! isInvalidMatchToken ) { const matchStatement = this . _parseMatchStatement ( ) ; if ( matchStatement ) { return matchStatement ; } } } } if ( this . _peekOperatorType ( ) === OperatorType . MatrixMultiply ) { return this . _parseDecorated ( ) ; } return this . _parseSimpleStatement ( ) ; } private _parseAsyncStatement ( ) { const asyncToken = this . _getKeywordToken ( KeywordType . Async ) ; switch ( this . _peekKeywordType ( ) ) { case KeywordType . Def : return this . _parseFunctionDef ( asyncToken ) ; case KeywordType . With : return this . _parseWithStatement ( asyncToken ) ; case KeywordType . For : return this . _parseForStatement ( asyncToken ) ; } this . _addError ( Localizer . Diagnostic . unexpectedAsyncToken ( ) , asyncToken ) ; return undefined ; } private _parseMatchStatement ( ) { let smellsLikeMatchStatement = false ; this . _suppressErrors ( ( ) => { const curTokenIndex = this . _tokenIndex ; this . _getKeywordToken ( KeywordType . Match ) ; const expression = this . _parseTestOrStarListAsExpression ( true , ErrorExpressionCategory . MissingPatternSubject , Localizer . Diagnostic . expectedReturnExpr ( ) ) ; smellsLikeMatchStatement = expression . nodeType !== ParseNodeType . Error && this . _peekToken ( ) . type === TokenType . Colon ; this . _tokenIndex = curTokenIndex ; } ) ; if ( ! smellsLikeMatchStatement ) { return undefined ; } const matchToken = this . _getKeywordToken ( KeywordType . Match ) ; const subjectExpression = this . _parseTestOrStarListAsExpression ( true , ErrorExpressionCategory . MissingPatternSubject , Localizer . Diagnostic . expectedReturnExpr ( ) ) ; const matchNode = MatchNode . create ( matchToken , subjectExpression ) ; const nextToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfType ( TokenType . Colon ) ) { this . _addError ( Localizer . Diagnostic . expectedColon ( ) , nextToken ) ; if ( this . _consumeTokensUntilType ( [ TokenType . NewLine , TokenType . Colon ] ) ) { this . _getNextToken ( ) ; } } else if ( ! this . _consumeTokenIfType ( TokenType . NewLine ) ) { this . _addError ( Localizer . Diagnostic . expectedNewline ( ) , nextToken ) ; } else { const possibleIndent = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfType ( TokenType . Indent ) ) { this . _addError ( Localizer . Diagnostic . expectedIndentedBlock ( ) , this . _peekToken ( ) ) ; } else { const indentToken = possibleIndent as IndentToken ; if ( indentToken . isIndentAmbiguous ) { this . _addError ( Localizer . Diagnostic . inconsistentTabs ( ) , indentToken ) ; } } while ( true ) { const nextToken = this . _peekToken ( ) ; if ( nextToken . type === TokenType . Indent ) { this . _getNextToken ( ) ; const indentToken = nextToken as IndentToken ; if ( indentToken . isIndentAmbiguous ) { this . _addError ( Localizer . Diagnostic . inconsistentTabs ( ) , indentToken ) ; } else { this . _addError ( Localizer . Diagnostic . unexpectedIndent ( ) , nextToken ) ; } } const caseStatement = this . _parseCaseStatement ( ) ; if ( ! caseStatement ) { if ( this . _consumeTokensUntilType ( [ TokenType . NewLine , TokenType . Colon ] ) ) { this . _getNextToken ( ) ; } } else { caseStatement . parent = matchNode ; matchNode . cases . push ( caseStatement ) ; } const dedentToken = this . _peekToken ( ) as DedentToken ; if ( this . _consumeTokenIfType ( TokenType . Dedent ) ) { if ( ! dedentToken . matchesIndent ) { this . _addError ( Localizer . Diagnostic . inconsistentIndent ( ) , dedentToken ) ; } if ( dedentToken . isDedentAmbiguous ) { this . _addError ( Localizer . Diagnostic . inconsistentTabs ( ) , dedentToken ) ; } break ; } if ( this . _peekTokenType ( ) === TokenType . EndOfStream ) { break ; } } if ( matchNode . cases . length > 0 ) { extendRange ( matchNode , matchNode . cases [ matchNode . cases . length - 1 ] ) ; } else { this . _addError ( Localizer . Diagnostic . zeroCaseStatementsFound ( ) , matchToken ) ; } } if ( this . _getLanguageVersion ( ) < PythonVersion . V3_10 ) { this . _addError ( Localizer . Diagnostic . matchIncompatible ( ) , matchToken ) ; } for ( let i = 0 ; i < matchNode . cases . length - 1 ; i ++ ) { const caseNode = matchNode . cases [ i ] ; if ( ! caseNode . guardExpression && caseNode . isIrrefutable ) { this . _addError ( Localizer . Diagnostic . casePatternIsIrrefutable ( ) , caseNode . pattern ) ; } } return matchNode ; } private _parseCaseStatement ( ) { const caseToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfKeyword ( KeywordType . Case ) ) { this . _addError ( Localizer . Diagnostic . expectedCase ( ) , caseToken ) ; return undefined ; } const patternList = this . _parsePatternSequence ( ) ; let casePattern ; if ( patternList . parseError ) { casePattern = patternList . parseError ; } else if ( patternList . list . length === 0 ) { this . _addError ( Localizer . Diagnostic . expectedPatternExpr ( ) , this . _peekToken ( ) ) ; casePattern = ErrorNode . create ( caseToken , ErrorExpressionCategory . MissingPattern ) ; } else if ( patternList . list . length === 1 && ! patternList . trailingComma ) { const pattern = patternList . list [ 0 ] . orPatterns [ 0 ] ; if ( pattern . nodeType === ParseNodeType . PatternCapture && pattern . isStar ) { casePattern = PatternSequenceNode . create ( patternList . list [ 0 ] , patternList . list ) ; } else { casePattern = patternList . list [ 0 ] ; } } else { casePattern = PatternSequenceNode . create ( patternList . list [ 0 ] , patternList . list ) ; } let guardExpression ; if ( this . _consumeTokenIfKeyword ( KeywordType . If ) ) { guardExpression = this . _parseTestExpression ( true ) ; } const suite = this . _parseSuite ( this . _isInFunction ) ; return CaseNode . create ( caseToken , casePattern , this . _isPatternIrrefutable ( casePattern ) , guardExpression , suite ) ; } private _isPatternIrrefutable ( node ) { if ( node . nodeType === ParseNodeType . PatternCapture ) { return true ; } if ( node . nodeType === ParseNodeType . PatternAs ) { return node . orPatterns . some ( ( pattern ) => this . _isPatternIrrefutable ( pattern ) ) ; } return false ; } private _getPatternTargetNames ( node , nameMap ) { switch ( node . nodeType ) { case ParseNodeType . PatternSequence : { node . entries . forEach ( ( subpattern ) => { this . _getPatternTargetNames ( subpattern , nameMap ) ; } ) ; break ; } case ParseNodeType . PatternClass : { node . arguments . forEach ( ( arg ) => { this . _getPatternTargetNames ( arg . pattern , nameMap ) ; } ) ; break ; } case ParseNodeType . PatternAs : { if ( node . target ) { nameMap . set ( node . target . value , true ) ; } node . orPatterns . forEach ( ( subpattern ) => { this . _getPatternTargetNames ( subpattern , nameMap ) ; } ) ; break ; } case ParseNodeType . PatternCapture : { if ( ! node . isWildcard ) { nameMap . set ( node . target . value , true ) ; } break ; } case ParseNodeType . PatternMapping : { node . entries . forEach ( ( mapEntry ) => { if ( mapEntry . nodeType === ParseNodeType . PatternMappingExpandEntry ) { nameMap . set ( mapEntry . target . value , true ) ; } else { this . _getPatternTargetNames ( mapEntry . keyPattern , nameMap ) ; this . _getPatternTargetNames ( mapEntry . valuePattern , nameMap ) ; } } ) ; break ; } case ParseNodeType . PatternLiteral : case ParseNodeType . PatternValue : case ParseNodeType . Error : { break ; } } } private _parsePatternSequence ( ) { const patternList = this . _parseExpressionListGeneric ( ( ) => this . _parsePatternAs ( ) ) ; const starEntries = patternList . list . filter ( ( entry ) => entry . orPatterns . length === 1 && entry . orPatterns [ 0 ] . nodeType === ParseNodeType . PatternCapture && entry . orPatterns [ 0 ] . isStar ) ; if ( starEntries . length > 1 ) { this . _addError ( Localizer . Diagnostic . duplicateStarPattern ( ) , starEntries [ 1 ] . orPatterns [ 0 ] ) ; } const captureTargetMap = new Map < string , PatternAtomNode > ( ) ; patternList . list . forEach ( ( asPattern ) => { asPattern . orPatterns . forEach ( ( patternAtom ) => { if ( patternAtom . nodeType === ParseNodeType . PatternCapture && ! patternAtom . isStar && ! patternAtom . isWildcard ) { if ( captureTargetMap . has ( patternAtom . target . value ) ) { this . _addError ( Localizer . Diagnostic . duplicateCapturePatternTarget ( ) . format ( { name : patternAtom . target . value , } ) , patternAtom ) ; } else { captureTargetMap . set ( patternAtom . target . value , patternAtom ) ; } } } ) ; } ) ; return patternList ; } private _parsePatternAs ( ) { const orPatterns = [ ] ; while ( true ) { const patternAtom = this . _parsePatternAtom ( ) ; orPatterns . push ( patternAtom ) ; if ( ! this . _consumeTokenIfOperator ( OperatorType . BitwiseOr ) ) { break ; } } if ( orPatterns . length > 1 ) { orPatterns . forEach ( ( patternAtom ) => { if ( patternAtom . nodeType === ParseNodeType . PatternCapture && patternAtom . isStar ) { this . _addError ( Localizer . Diagnostic . starPatternInOrPattern ( ) , patternAtom ) ; } } ) ; } let target ; if ( this . _consumeTokenIfKeyword ( KeywordType . As ) ) { const nameToken = this . _getTokenIfIdentifier ( ) ; if ( nameToken ) { target = NameNode . create ( nameToken ) ; } else { this . _addError ( Localizer . Diagnostic . expectedNameAfterAs ( ) , this . _peekToken ( ) ) ; } } if ( target && orPatterns . length === 1 && orPatterns [ 0 ] . nodeType === ParseNodeType . PatternCapture && orPatterns [ 0 ] . isStar ) { this . _addError ( Localizer . Diagnostic . starPatternInAsPattern ( ) , orPatterns [ 0 ] ) ; } orPatterns . forEach ( ( orPattern , index ) => { if ( index < orPatterns . length - 1 && this . _isPatternIrrefutable ( orPattern ) ) { this . _addError ( Localizer . Diagnostic . orPatternIrrefutable ( ) , orPattern ) ; } } ) ; const fullNameMap = new Map < string , boolean > ( ) ; orPatterns . forEach ( ( orPattern ) => { this . _getPatternTargetNames ( orPattern , fullNameMap ) ; } ) ; orPatterns . forEach ( ( orPattern ) => { const localNameMap = new Map < string , boolean > ( ) ; this . _getPatternTargetNames ( orPattern , localNameMap ) ; if ( localNameMap . size < fullNameMap . size ) { const missingNames = Array . from ( fullNameMap . keys ( ) ) . filter ( ( name ) => ! localNameMap . has ( name ) ) ; const diag = new DiagnosticAddendum ( ) ; diag . addMessage ( Localizer . DiagnosticAddendum . orPatternMissingName ( ) . format ( { name : missingNames . map ( ( name ) => `"${ name }"` ) . join ( ', ' ) , } ) ) ; this . _addError ( Localizer . Diagnostic . orPatternMissingName ( ) + diag . getString ( ) , orPattern ) ; } } ) ; return PatternAsNode . create ( orPatterns , target ) ; } private _parsePatternAtom ( ) { const patternLiteral = this . _parsePatternLiteral ( ) ; if ( patternLiteral ) { return patternLiteral ; } const patternCaptureOrValue = this . _parsePatternCaptureOrValue ( ) ; if ( patternCaptureOrValue ) { const openParenToken = this . _peekToken ( ) ; if ( patternCaptureOrValue . nodeType === ParseNodeType . Error || ! this . _consumeTokenIfType ( TokenType . OpenParenthesis ) ) { return patternCaptureOrValue ; } const args = this . _parseClassPatternArgList ( ) ; const classNameExpr = patternCaptureOrValue . nodeType === ParseNodeType . PatternCapture ? patternCaptureOrValue . target : patternCaptureOrValue . expression ; const classPattern = PatternClassNode . create ( classNameExpr , args ) ; if ( ! this . _consumeTokenIfType ( TokenType . CloseParenthesis ) ) { this . _addError ( Localizer . Diagnostic . expectedCloseParen ( ) , openParenToken ) ; this . _consumeTokensUntilType ( [ TokenType . NewLine ] ) ; extendRange ( classPattern , this . _peekToken ( ) ) ; } return classPattern ; } const nextToken = this . _peekToken ( ) ; const nextOperator = this . _peekOperatorType ( ) ; if ( nextOperator === OperatorType . Multiply ) { const starToken = this . _getNextToken ( ) ; const identifierToken = this . _getTokenIfIdentifier ( ) ; if ( ! identifierToken ) { this . _addError ( Localizer . Diagnostic . expectedIdentifier ( ) , this . _peekToken ( ) ) ; return ErrorNode . create ( starToken , ErrorExpressionCategory . MissingExpression ) ; } else { return PatternCaptureNode . create ( NameNode . create ( identifierToken ) , starToken ) ; } } if ( nextToken . type === TokenType . OpenParenthesis || nextToken . type === TokenType . OpenBracket ) { const startToken = this . _getNextToken ( ) ; const patternList = this . _parsePatternSequence ( ) ; let casePattern ; if ( patternList . parseError ) { casePattern = patternList . parseError ; } else if ( patternList . list . length === 1 && ! patternList . trailingComma && startToken . type === TokenType . OpenParenthesis ) { const pattern = patternList . list [ 0 ] . orPatterns [ 0 ] ; if ( pattern . nodeType === ParseNodeType . PatternCapture && pattern . isStar ) { casePattern = PatternSequenceNode . create ( startToken , patternList . list ) ; } else { casePattern = patternList . list [ 0 ] ; } extendRange ( casePattern , nextToken ) ; } else { casePattern = PatternSequenceNode . create ( startToken , patternList . list ) ; } const endToken = this . _peekToken ( ) ; if ( this . _consumeTokenIfType ( nextToken . type === TokenType . OpenParenthesis ? TokenType . CloseParenthesis : TokenType . CloseBracket ) ) { extendRange ( casePattern , endToken ) ; } else { this . _addError ( nextToken . type === TokenType . OpenParenthesis ? Localizer . Diagnostic . expectedCloseParen ( ) : Localizer . Diagnostic . expectedCloseBracket ( ) , nextToken ) ; this . _consumeTokensUntilType ( [ TokenType . Colon , nextToken . type === TokenType . OpenParenthesis ? TokenType . CloseParenthesis : TokenType . CloseBracket , ] ) ; } return casePattern ; } else if ( nextToken . type === TokenType . OpenCurlyBrace ) { const firstToken = this . _getNextToken ( ) ; const mappingPattern = this . _parsePatternMapping ( firstToken ) ; const lastToken = this . _peekToken ( ) ; if ( this . _consumeTokenIfType ( TokenType . CloseCurlyBrace ) ) { extendRange ( mappingPattern , lastToken ) ; } else { this . _addError ( Localizer . Diagnostic . expectedCloseBrace ( ) , nextToken ) ; this . _consumeTokensUntilType ( [ TokenType . Colon , TokenType . CloseCurlyBrace ] ) ; } return mappingPattern ; } return this . _handleExpressionParseError ( ErrorExpressionCategory . MissingPattern , Localizer . Diagnostic . expectedPatternExpr ( ) ) ; } private _parseClassPatternArgList ( ) { const argList = [ ] ; let sawKeywordArg = false ; while ( true ) { const nextTokenType = this . _peekTokenType ( ) ; if ( nextTokenType === TokenType . CloseParenthesis || nextTokenType === TokenType . NewLine || nextTokenType === TokenType . EndOfStream ) { break ; } const arg = this . _parseClassPatternArgument ( ) ; if ( arg . name ) { sawKeywordArg = true ; } else if ( sawKeywordArg && ! arg . name ) { this . _addError ( Localizer . Diagnostic . positionArgAfterNamedArg ( ) , arg ) ; } argList . push ( arg ) ; if ( ! this . _consumeTokenIfType ( TokenType . Comma ) ) { break ; } } return argList ; } private _parseClassPatternArgument ( ) { const firstToken = this . _peekToken ( ) ; const secondToken = this . _peekToken ( 1 ) ; let keywordName ; if ( ( firstToken . type === TokenType . Identifier || firstToken . type === TokenType . Keyword ) && secondToken . type === TokenType . Operator && ( secondToken as OperatorToken ) . operatorType === OperatorType . Assign ) { this . _getNextToken ( ) ; keywordName = NameNode . create ( firstToken as IdentifierToken ) ; this . _getNextToken ( ) ; } const pattern = this . _parsePatternAs ( ) ; return PatternClassArgumentNode . create ( pattern , keywordName ) ; } private _parsePatternLiteral ( ) { const nextToken = this . _peekToken ( ) ; const nextOperator = this . _peekOperatorType ( ) ; if ( nextToken . type === TokenType . Number || nextOperator === OperatorType . Subtract ) { return this . _parsePatternLiteralNumber ( ) ; } if ( nextToken . type === TokenType . String ) { const stringList = this . _parseAtom ( ) as StringListNode ; assert ( stringList . nodeType === ParseNodeType . StringList ) ; stringList . strings . forEach ( ( stringAtom ) => { if ( stringAtom . token . flags & StringTokenFlags . Format ) { this . _addError ( Localizer . Diagnostic . formatStringInPattern ( ) , stringAtom ) ; } } ) ; return PatternLiteralNode . create ( stringList ) ; } if ( nextToken . type === TokenType . Keyword ) { const keywordToken = nextToken as KeywordToken ; if ( keywordToken . keywordType === KeywordType . False || keywordToken . keywordType === KeywordType . True || keywordToken . keywordType === KeywordType . None ) { return PatternLiteralNode . create ( this . _parseAtom ( ) ) ; } } return undefined ; } private _parsePatternLiteralNumber ( ) { const expression = this . _parseArithmeticExpression ( ) ; let realValue ; let imagValue ; if ( expression . nodeType === ParseNodeType . BinaryOperation ) { if ( expression . operator === OperatorType . Subtract || expression . operator === OperatorType . Add ) { realValue = expression . leftExpression ; imagValue = expression . rightExpression ; } } else { realValue = expression ; } if ( realValue ) { if ( realValue . nodeType === ParseNodeType . UnaryOperation && realValue . operator === OperatorType . Subtract ) { realValue = realValue . expression ; } if ( realValue . nodeType !== ParseNodeType . Number || ( imagValue !== undefined && realValue . isImaginary ) ) { this . _addError ( Localizer . Diagnostic . expectedComplexNumberLiteral ( ) , expression ) ; imagValue = undefined ; } } if ( imagValue ) { if ( imagValue . nodeType === ParseNodeType . UnaryOperation && imagValue . operator === OperatorType . Subtract ) { imagValue = imagValue . expression ; } if ( imagValue . nodeType !== ParseNodeType . Number || ! imagValue . isImaginary ) { this . _addError ( Localizer . Diagnostic . expectedComplexNumberLiteral ( ) , expression ) ; } } return PatternLiteralNode . create ( expression ) ; } private _parsePatternMapping ( firstToken ) { const itemList = this . _parseExpressionListGeneric ( ( ) => this . _parsePatternMappingItem ( ) ) ; if ( itemList . list . length > 0 ) { const starStarEntries = itemList . list . filter ( ( entry ) => entry . nodeType === ParseNodeType . PatternMappingExpandEntry ) ; if ( starStarEntries . length > 1 ) { this . _addError ( Localizer . Diagnostic . duplicateStarStarPattern ( ) , starStarEntries [ 1 ] ) ; } return PatternMappingNode . create ( firstToken , itemList . list ) ; } return itemList . parseError || ErrorNode . create ( this . _peekToken ( ) , ErrorExpressionCategory . MissingPattern ) ; } private _parsePatternMappingItem ( ) { let keyExpression ; const doubleStar = this . _peekToken ( ) ; if ( this . _consumeTokenIfOperator ( OperatorType . Power ) ) { const identifierToken = this . _getTokenIfIdentifier ( ) ; if ( ! identifierToken ) { this . _addError ( Localizer . Diagnostic . expectedIdentifier ( ) , this . _peekToken ( ) ) ; return ErrorNode . create ( this . _peekToken ( ) , ErrorExpressionCategory . MissingPattern ) ; } const nameNode = NameNode . create ( identifierToken ) ; if ( identifierToken . value === '_' ) { this . _addError ( Localizer . Diagnostic . starStarWildcardNotAllowed ( ) , nameNode ) ; } return PatternMappingExpandEntryNode . create ( doubleStar , nameNode ) ; } const patternLiteral = this . _parsePatternLiteral ( ) ; if ( patternLiteral ) { keyExpression = patternLiteral ; } else { const patternCaptureOrValue = this . _parsePatternCaptureOrValue ( ) ; if ( patternCaptureOrValue ) { if ( patternCaptureOrValue . nodeType === ParseNodeType . PatternValue ) { keyExpression = patternCaptureOrValue ; } else { this . _addError ( Localizer . Diagnostic . expectedPatternValue ( ) , patternCaptureOrValue ) ; keyExpression = ErrorNode . create ( this . _peekToken ( ) , ErrorExpressionCategory . MissingPattern ) ; } } } if ( ! keyExpression ) { this . _addError ( Localizer . Diagnostic . expectedPatternExpr ( ) , this . _peekToken ( ) ) ; keyExpression = ErrorNode . create ( this . _peekToken ( ) , ErrorExpressionCategory . MissingPattern ) ; } let valuePattern ; if ( ! this . _consumeTokenIfType ( TokenType . Colon ) ) { this . _addError ( Localizer . Diagnostic . expectedColon ( ) , this . _peekToken ( ) ) ; valuePattern = ErrorNode . create ( this . _peekToken ( ) , ErrorExpressionCategory . MissingPattern ) ; } else { valuePattern = this . _parsePatternAs ( ) ; } return PatternMappingKeyEntryNode . create ( keyExpression , valuePattern ) ; } private _parsePatternCaptureOrValue ( ) { const nextToken = this . _peekToken ( ) ; if ( nextToken . type === TokenType . Identifier || nextToken . type === TokenType . Keyword ) { let nameOrMember ; while ( true ) { const identifierToken = this . _getTokenIfIdentifier ( ) ; if ( identifierToken ) { const nameNode = NameNode . create ( identifierToken ) ; nameOrMember = nameOrMember ? MemberAccessNode . create ( nameOrMember , nameNode ) : nameNode ; } else { this . _addError ( Localizer . Diagnostic . expectedIdentifier ( ) , this . _peekToken ( ) ) ; break ; } if ( ! this . _consumeTokenIfType ( TokenType . Dot ) ) { break ; } } if ( ! nameOrMember ) { this . _addError ( Localizer . Diagnostic . expectedIdentifier ( ) , this . _peekToken ( ) ) ; return ErrorNode . create ( this . _peekToken ( ) , ErrorExpressionCategory . MissingPattern ) ; } if ( nameOrMember . nodeType === ParseNodeType . MemberAccess ) { return PatternValueNode . create ( nameOrMember ) ; } return PatternCaptureNode . create ( nameOrMember ) ; } return undefined ; } private _parseIfStatement ( keywordType = KeywordType . If ) { const ifOrElifToken = this . _getKeywordToken ( keywordType ) ; const test = this . _parseTestExpression ( true ) ; const suite = this . _parseSuite ( this . _isInFunction ) ; const ifNode = IfNode . create ( ifOrElifToken , test , suite ) ; if ( this . _consumeTokenIfKeyword ( KeywordType . Else ) ) { ifNode . elseSuite = this . _parseSuite ( this . _isInFunction ) ; ifNode . elseSuite . parent = ifNode ; extendRange ( ifNode , ifNode . elseSuite ) ; } else if ( this . _peekKeywordType ( ) === KeywordType . Elif ) { ifNode . elseSuite = this . _parseIfStatement ( KeywordType . Elif ) ; ifNode . elseSuite . parent = ifNode ; extendRange ( ifNode , ifNode . elseSuite ) ; } return ifNode ; } private _parseLoopSuite ( ) { const wasInLoop = this . _isInLoop ; const wasInFinally = this . _isInFinally ; this . _isInLoop = true ; this . _isInFinally = false ; const suite = this . _parseSuite ( this . _isInFunction ) ; this . _isInLoop = wasInLoop ; this . _isInFinally = wasInFinally ; return suite ; } private _parseSuite ( isFunction = false , skipBody = false , postColonCallback ? ) { const nextToken = this . _peekToken ( ) ; const suite = SuiteNode . create ( nextToken ) ; if ( ! this . _consumeTokenIfType ( TokenType . Colon ) ) { this . _addError ( Localizer . Diagnostic . expectedColon ( ) , nextToken ) ; if ( this . _consumeTokensUntilType ( [ TokenType . NewLine , TokenType . Colon ] ) ) { if ( this . _peekTokenType ( ) === TokenType . Colon ) { this . _getNextToken ( ) ; } else if ( this . _peekToken ( 1 ) . type !== TokenType . Indent ) { this . _getNextToken ( ) ; return suite ; } } } if ( skipBody ) { if ( this . _consumeTokenIfType ( TokenType . NewLine ) ) { let indent = 0 ; while ( true ) { const nextToken = this . _getNextToken ( ) ; if ( nextToken . type === TokenType . Indent ) { indent ++ ; } if ( nextToken . type === TokenType . Dedent ) { if ( ( nextToken as DedentToken ) . isDedentAmbiguous ) { this . _addError ( Localizer . Diagnostic . inconsistentTabs ( ) , nextToken ) ; } indent -- ; if ( indent === 0 ) { break ; } } if ( nextToken . type === TokenType . EndOfStream ) { break ; } } } else { this . _parseSimpleStatement ( ) ; } if ( this . _tokenIndex > 0 ) { extendRange ( suite , this . _tokenizerOutput ! . tokens . getItemAt ( this . _tokenIndex - 1 ) ) ; } return suite ; } if ( postColonCallback ) { postColonCallback ( ) ; } const wasFunction = this . _isInFunction ; this . _isInFunction = isFunction ; if ( this . _consumeTokenIfType ( TokenType . NewLine ) ) { if ( postColonCallback ) { postColonCallback ( ) ; } const possibleIndent = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfType ( TokenType . Indent ) ) { this . _addError ( Localizer . Diagnostic . expectedIndentedBlock ( ) , this . _peekToken ( ) ) ; } else { const indentToken = possibleIndent as IndentToken ; if ( indentToken . isIndentAmbiguous ) { this . _addError ( Localizer . Diagnostic . inconsistentTabs ( ) , indentToken ) ; } } while ( true ) { const nextToken = this . _peekToken ( ) ; if ( nextToken . type === TokenType . Indent ) { this . _getNextToken ( ) ; const indentToken = nextToken as IndentToken ; if ( indentToken . isIndentAmbiguous ) { this . _addError ( Localizer . Diagnostic . inconsistentTabs ( ) , indentToken ) ; } else { this . _addError ( Localizer . Diagnostic . unexpectedIndent ( ) , nextToken ) ; } } const statement = this . _parseStatement ( ) ; if ( ! statement ) { this . _consumeTokensUntilType ( [ TokenType . NewLine ] ) ; } else { statement . parent = suite ; suite . statements . push ( statement ) ; } const dedentToken = this . _peekToken ( ) as DedentToken ; if ( this . _consumeTokenIfType ( TokenType . Dedent ) ) { if ( ! dedentToken . matchesIndent ) { this . _addError ( Localizer . Diagnostic . inconsistentIndent ( ) , dedentToken ) ; } if ( dedentToken . isDedentAmbiguous ) { this . _addError ( Localizer . Diagnostic . inconsistentTabs ( ) , dedentToken ) ; } break ; } if ( this . _peekTokenType ( ) === TokenType . EndOfStream ) { break ; } } } else { const simpleStatement = this . _parseSimpleStatement ( ) ; suite . statements . push ( simpleStatement ) ; simpleStatement . parent = suite ; } if ( suite . statements . length > 0 ) { extendRange ( suite , suite . statements [ suite . statements . length - 1 ] ) ; } this . _isInFunction = wasFunction ; return suite ; } private _parseForStatement ( asyncToken ? ) { const forToken = this . _getKeywordToken ( KeywordType . For ) ; const targetExpr = this . _parseExpressionListAsPossibleTuple ( ErrorExpressionCategory . MissingExpression , Localizer . Diagnostic . expectedExpr ( ) , forToken ) ; let seqExpr ; let forSuite ; let elseSuite ; if ( ! this . _consumeTokenIfKeyword ( KeywordType . In ) ) { seqExpr = this . _handleExpressionParseError ( ErrorExpressionCategory . MissingIn , Localizer . Diagnostic . expectedIn ( ) ) ; forSuite = SuiteNode . create ( this . _peekToken ( ) ) ; } else { seqExpr = this . _parseTestListAsExpression ( ErrorExpressionCategory . MissingExpression , Localizer . Diagnostic . expectedInExpr ( ) ) ; forSuite = this . _parseLoopSuite ( ) ; if ( this . _consumeTokenIfKeyword ( KeywordType . Else ) ) { elseSuite = this . _parseSuite ( this . _isInFunction ) ; } } const forNode = ForNode . create ( forToken , targetExpr , seqExpr , forSuite ) ; forNode . elseSuite = elseSuite ; if ( elseSuite ) { extendRange ( forNode , elseSuite ) ; elseSuite . parent = forNode ; } if ( asyncToken ) { forNode . isAsync = true ; forNode . asyncToken = asyncToken ; extendRange ( forNode , asyncToken ) ; } return forNode ; } private _tryParseListComprehension ( target ) { const compFor = this . _tryParseCompForStatement ( ) ; if ( ! compFor ) { return undefined ; } if ( target . nodeType === ParseNodeType . Unpack ) { this . _addError ( Localizer . Diagnostic . unpackIllegalInComprehension ( ) , target ) ; } else if ( target . nodeType === ParseNodeType . DictionaryExpandEntry ) { this . _addError ( Localizer . Diagnostic . dictExpandIllegalInComprehension ( ) , target ) ; } const listCompNode = ListComprehensionNode . create ( target ) ; const compList = [ compFor ] ; while ( true ) { const compIter = this . _tryParseCompForStatement ( ) || this . _tryParseCompIfStatement ( ) ; if ( ! compIter ) { break ; } compIter . parent = listCompNode ; compList . push ( compIter ) ; } listCompNode . comprehensions = compList ; if ( compList . length > 0 ) { compList . forEach ( ( comp ) => { comp . parent = listCompNode ; } ) ; extendRange ( listCompNode , compList [ compList . length - 1 ] ) ; } return listCompNode ; } private _tryParseCompForStatement ( ) { const startTokenKeywordType = this . _peekKeywordType ( ) ; if ( startTokenKeywordType === KeywordType . Async ) { const nextToken = this . _peekToken ( 1 ) as KeywordToken ; if ( nextToken . type !== TokenType . Keyword || nextToken . keywordType !== KeywordType . For ) { return undefined ; } } else if ( startTokenKeywordType !== KeywordType . For ) { return undefined ; } let asyncToken ; if ( this . _peekKeywordType ( ) === KeywordType . Async ) { asyncToken = this . _getKeywordToken ( KeywordType . Async ) ; } const forToken = this . _getKeywordToken ( KeywordType . For ) ; const targetExpr = this . _parseExpressionListAsPossibleTuple ( ErrorExpressionCategory . MissingExpression , Localizer . Diagnostic . expectedExpr ( ) , forToken ) ; let seqExpr ; if ( ! this . _consumeTokenIfKeyword ( KeywordType . In ) ) { seqExpr = this . _handleExpressionParseError ( ErrorExpressionCategory . MissingIn , Localizer . Diagnostic . expectedIn ( ) ) ; } else { this . _disallowAssignmentExpression ( ( ) => { seqExpr = this . _parseOrTest ( ) ; } ) ; } const compForNode = ListComprehensionForNode . create ( asyncToken || forToken , targetExpr , seqExpr ! ) ; if ( asyncToken ) { compForNode . isAsync = true ; compForNode . asyncToken = asyncToken ; } return compForNode ; } private _tryParseCompIfStatement ( ) { if ( this . _peekKeywordType ( ) !== KeywordType . If ) { return undefined ; } const ifToken = this . _getKeywordToken ( KeywordType . If ) ; const ifExpr = this . _tryParseLambdaExpression ( ) || this . _parseAssignmentExpression ( ) ; const compIfNode = ListComprehensionIfNode . create ( ifToken , ifExpr ) ; return compIfNode ; } private _parseWhileStatement ( ) { const whileToken = this . _getKeywordToken ( KeywordType . While ) ; const whileNode = WhileNode . create ( whileToken , this . _parseTestExpression ( true ) , this . _parseLoopSuite ( ) ) ; if ( this . _consumeTokenIfKeyword ( KeywordType . Else ) ) { whileNode . elseSuite = this . _parseSuite ( this . _isInFunction ) ; whileNode . elseSuite . parent = whileNode ; extendRange ( whileNode , whileNode . elseSuite ) ; } return whileNode ; } private _parseTryStatement ( ) { const tryToken = this . _getKeywordToken ( KeywordType . Try ) ; const trySuite = this . _parseSuite ( this . _isInFunction ) ; const tryNode = TryNode . create ( tryToken , trySuite ) ; let sawCatchAllExcept = false ; while ( true ) { const exceptToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfKeyword ( KeywordType . Except ) ) { break ; } const possibleStarToken = this . _peekToken ( ) ; let isExceptGroup = false ; if ( this . _consumeTokenIfOperator ( OperatorType . Multiply ) ) { if ( this . _getLanguageVersion ( ) < PythonVersion . V3_11 && ! this . _parseOptions . isStubFile ) { this . _addError ( Localizer . Diagnostic . exceptionGroupIncompatible ( ) , possibleStarToken ) ; } isExceptGroup = true ; } let typeExpr ; let symbolName ; if ( this . _peekTokenType ( ) !== TokenType . Colon ) { typeExpr = this . _parseTestExpression ( true ) ; if ( this . _consumeTokenIfKeyword ( KeywordType . As ) ) { symbolName = this . _getTokenIfIdentifier ( ) ; if ( ! symbolName ) { this . _addError ( Localizer . Diagnostic . expectedNameAfterAs ( ) , this . _peekToken ( ) ) ; } } else { const peekToken = this . _peekToken ( ) ; if ( this . _consumeTokenIfType ( TokenType . Comma ) ) { this . _addError ( Localizer . Diagnostic . expectedAsAfterException ( ) , peekToken ) ; this . _parseTestExpression ( false ) ; } } } if ( ! typeExpr ) { if ( sawCatchAllExcept ) { this . _addError ( Localizer . Diagnostic . duplicateCatchAll ( ) , exceptToken ) ; } sawCatchAllExcept = true ; } else { if ( sawCatchAllExcept ) { this . _addError ( Localizer . Diagnostic . namedExceptAfterCatchAll ( ) , typeExpr ) ; } } const exceptSuite = this . _parseSuite ( this . _isInFunction ) ; const exceptNode = ExceptNode . create ( exceptToken , exceptSuite , isExceptGroup ) ; if ( typeExpr ) { exceptNode . typeExpression = typeExpr ; exceptNode . typeExpression . parent = exceptNode ; } if ( symbolName ) { exceptNode . name = NameNode . create ( symbolName ) ; exceptNode . name . parent = exceptNode ; } tryNode . exceptClauses . push ( exceptNode ) ; exceptNode . parent = tryNode ; } if ( tryNode . exceptClauses . length > 0 ) { extendRange ( tryNode , tryNode . exceptClauses [ tryNode . exceptClauses . length - 1 ] ) ; if ( this . _consumeTokenIfKeyword ( KeywordType . Else ) ) { tryNode . elseSuite = this . _parseSuite ( this . _isInFunction ) ; tryNode . elseSuite . parent = tryNode ; extendRange ( tryNode , tryNode . elseSuite ) ; } } if ( this . _consumeTokenIfKeyword ( KeywordType . Finally ) ) { tryNode . finallySuite = this . _parseSuite ( this . _isInFunction ) ; tryNode . finallySuite . parent = tryNode ; extendRange ( tryNode , tryNode . finallySuite ) ; } if ( ! tryNode . finallySuite && tryNode . exceptClauses . length === 0 ) { this . _addError ( Localizer . Diagnostic . tryWithoutExcept ( ) , tryToken ) ; } return tryNode ; } private _parseFunctionDef ( asyncToken ? , decorators ? ) { const defToken = this . _getKeywordToken ( KeywordType . Def ) ; const nameToken = this . _getTokenIfIdentifier ( ) ; if ( ! nameToken ) { this . _addError ( Localizer . Diagnostic . expectedFunctionName ( ) , defToken ) ; return ErrorNode . create ( defToken , ErrorExpressionCategory . MissingFunctionParameterList , undefined , decorators ) ; } const openParenToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfType ( TokenType . OpenParenthesis ) ) { this . _addError ( Localizer . Diagnostic . expectedOpenParen ( ) , this . _peekToken ( ) ) ; return ErrorNode . create ( nameToken , ErrorExpressionCategory . MissingFunctionParameterList , NameNode . create ( nameToken ) , decorators ) ; } const paramList = this . _parseVarArgsList ( TokenType . CloseParenthesis , true ) ; if ( ! this . _consumeTokenIfType ( TokenType . CloseParenthesis ) ) { this . _addError ( Localizer . Diagnostic . expectedCloseParen ( ) , openParenToken ) ; this . _consumeTokensUntilType ( [ TokenType . Colon ] ) ; } let returnType ; if ( this . _consumeTokenIfType ( TokenType . Arrow ) ) { returnType = this . _parseTypeAnnotation ( ) ; } let functionTypeAnnotationToken ; const suite = this . _parseSuite ( true , this . _parseOptions . skipFunctionAndClassBody , ( ) => { if ( ! functionTypeAnnotationToken ) { functionTypeAnnotationToken = this . _getTypeAnnotationCommentText ( ) ; } } ) ; const functionNode = FunctionNode . create ( defToken , NameNode . create ( nameToken ) , suite ) ; if ( asyncToken ) { functionNode . isAsync = true ; extendRange ( functionNode , asyncToken ) ; } functionNode . parameters = paramList ; paramList . forEach ( ( param ) => { param . parent = functionNode ; } ) ; if ( decorators ) { functionNode . decorators = decorators ; decorators . forEach ( ( decorator ) => { decorator . parent = functionNode ; } ) ; if ( decorators . length > 0 ) { extendRange ( functionNode , decorators [ 0 ] ) ; } } if ( returnType ) { functionNode . returnTypeAnnotation = returnType ; functionNode . returnTypeAnnotation . parent = functionNode ; extendRange ( functionNode , returnType ) ; } if ( functionTypeAnnotationToken ) { this . _parseFunctionTypeAnnotationComment ( functionTypeAnnotationToken , functionNode ) ; } return functionNode ; } private _parseVarArgsList ( terminator , allowAnnotations ) { const paramMap = new Map < string , string > ( ) ; const paramList = [ ] ; let sawDefaultParam = false ; let reportedNonDefaultParamErr = false ; let sawKeywordOnlySeparator = false ; let sawPositionOnlySeparator = false ; let sawArgs = false ; let sawKwArgs = false ; while ( true ) { if ( this . _peekTokenType ( ) === terminator ) { break ; } const param = this . _parseParameter ( allowAnnotations ) ; if ( ! param ) { this . _consumeTokensUntilType ( [ terminator ] ) ; break ; } if ( param . name ) { const name = param . name . value ; if ( paramMap . has ( name ) ) { this . _addError ( Localizer . Diagnostic . duplicateParam ( ) . format ( { name } ) , param . name ) ; } else { paramMap . set ( name , name ) ; } } else if ( param . category === ParameterCategory . Simple ) { if ( paramList . length === 0 ) { this . _addError ( Localizer . Diagnostic . positionOnlyFirstParam ( ) , param ) ; } } if ( param . category === ParameterCategory . Simple ) { if ( ! param . name ) { if ( sawPositionOnlySeparator ) { this . _addError ( Localizer . Diagnostic . duplicatePositionOnly ( ) , param ) ; } else if ( sawKeywordOnlySeparator ) { this . _addError ( Localizer . Diagnostic . positionOnlyAfterKeywordOnly ( ) , param ) ; } else if ( sawArgs ) { this . _addError ( Localizer . Diagnostic . positionOnlyAfterArgs ( ) , param ) ; } sawPositionOnlySeparator = true ; } else { if ( param . defaultValue ) { sawDefaultParam = true ; } else if ( sawDefaultParam && ! sawKeywordOnlySeparator && ! sawArgs ) { if ( ! reportedNonDefaultParamErr ) { this . _addError ( Localizer . Diagnostic . nonDefaultAfterDefault ( ) , param ) ; reportedNonDefaultParamErr = true ; } } } } paramList . push ( param ) ; if ( param . category === ParameterCategory . VarArgList ) { if ( ! param . name ) { if ( sawKeywordOnlySeparator ) { this . _addError ( Localizer . Diagnostic . duplicateKeywordOnly ( ) , param ) ; } else if ( sawArgs ) { this . _addError ( Localizer . Diagnostic . keywordOnlyAfterArgs ( ) , param ) ; } sawKeywordOnlySeparator = true ; } else { if ( sawArgs ) { this . _addError ( Localizer . Diagnostic . duplicateArgsParam ( ) , param ) ; } sawArgs = true ; } } if ( param . category === ParameterCategory . VarArgDictionary ) { if ( sawKwArgs ) { this . _addError ( Localizer . Diagnostic . duplicateKwargsParam ( ) , param ) ; } sawKwArgs = true ; } else if ( sawKwArgs ) { this . _addError ( Localizer . Diagnostic . paramAfterKwargsParam ( ) , param ) ; } const foundComma = this . _consumeTokenIfType ( TokenType . Comma ) ; if ( allowAnnotations && ! param . typeAnnotation ) { const typeAnnotationComment = this . _parseVariableTypeAnnotationComment ( ) ; if ( typeAnnotationComment ) { param . typeAnnotationComment = typeAnnotationComment ; param . typeAnnotationComment . parent = param ; extendRange ( param , param . typeAnnotationComment ) ; } } if ( ! foundComma ) { break ; } } if ( paramList . length > 0 ) { const lastParam = paramList [ paramList . length - 1 ] ; if ( lastParam . category === ParameterCategory . VarArgList && ! lastParam . name ) { this . _addError ( Localizer . Diagnostic . expectedNamedParameter ( ) , lastParam ) ; } } return paramList ; } private _parseParameter ( allowAnnotations ) { let starCount = 0 ; let slashCount = 0 ; const firstToken = this . _peekToken ( ) ; if ( this . _consumeTokenIfOperator ( OperatorType . Multiply ) ) { starCount = 1 ; } else if ( this . _consumeTokenIfOperator ( OperatorType . Power ) ) { starCount = 2 ; } else if ( this . _consumeTokenIfOperator ( OperatorType . Divide ) ) { if ( this . _getLanguageVersion ( ) < PythonVersion . V3_8 && ! this . _parseOptions . isStubFile ) { this . _addError ( Localizer . Diagnostic . positionOnlyIncompatible ( ) , firstToken ) ; } slashCount = 1 ; } const paramName = this . _getTokenIfIdentifier ( ) ; if ( ! paramName ) { if ( starCount === 1 ) { const paramNode = ParameterNode . create ( firstToken , ParameterCategory . VarArgList ) ; return paramNode ; } else if ( slashCount === 1 ) { const paramNode = ParameterNode . create ( firstToken , ParameterCategory . Simple ) ; return paramNode ; } if ( this . _peekTokenType ( ) === TokenType . OpenParenthesis ) { const sublistStart = this . _getNextToken ( ) ; if ( this . _consumeTokensUntilType ( [ TokenType . CloseParenthesis ] ) ) { this . _getNextToken ( ) ; } this . _addError ( Localizer . Diagnostic . sublistParamsIncompatible ( ) , sublistStart ) ; } else { this . _addError ( Localizer . Diagnostic . expectedParamName ( ) , this . _peekToken ( ) ) ; } } let paramType = ParameterCategory . Simple ; if ( starCount === 1 ) { paramType = ParameterCategory . VarArgList ; } else if ( starCount === 2 ) { paramType = ParameterCategory . VarArgDictionary ; } const paramNode = ParameterNode . create ( firstToken , paramType ) ; if ( paramName ) { paramNode . name = NameNode . create ( paramName ) ; paramNode . name . parent = paramNode ; extendRange ( paramNode , paramName ) ; } if ( allowAnnotations && this . _consumeTokenIfType ( TokenType . Colon ) ) { paramNode . typeAnnotation = this . _parseTypeAnnotation ( ) ; paramNode . typeAnnotation . parent = paramNode ; extendRange ( paramNode , paramNode . typeAnnotation ) ; } if ( this . _consumeTokenIfOperator ( OperatorType . Assign ) ) { paramNode . defaultValue = this . _parseTestExpression ( false ) ; paramNode . defaultValue . parent = paramNode ; extendRange ( paramNode , paramNode . defaultValue ) ; if ( starCount > 0 ) { this . _addError ( Localizer . Diagnostic . defaultValueNotAllowed ( ) , paramNode . defaultValue ) ; } } return paramNode ; } private _parseWithStatement ( asyncToken ? ) { const withToken = this . _getKeywordToken ( KeywordType . With ) ; let withItemList = [ ] ; const possibleParen = this . _peekToken ( ) ; let isParenthesizedWithItemList = false ; if ( possibleParen . type === TokenType . OpenParenthesis ) { const openParenTokenIndex = this . _tokenIndex ; this . _suppressErrors ( ( ) => { this . _getNextToken ( ) ; while ( true ) { withItemList . push ( this . _parseWithItem ( ) ) ; if ( ! this . _consumeTokenIfType ( TokenType . Comma ) ) { break ; } if ( this . _peekToken ( ) . type === TokenType . CloseParenthesis ) { break ; } } if ( this . _peekToken ( ) . type === TokenType . CloseParenthesis && this . _peekToken ( 1 ) . type === TokenType . Colon ) { isParenthesizedWithItemList = withItemList . length !== 1 || withItemList [ 0 ] . target !== undefined ; } this . _tokenIndex = openParenTokenIndex ; withItemList = [ ] ; } ) ; } if ( isParenthesizedWithItemList ) { this . _consumeTokenIfType ( TokenType . OpenParenthesis ) ; if ( this . _getLanguageVersion ( ) < PythonVersion . V3_9 ) { this . _addError ( Localizer . Diagnostic . parenthesizedContextManagerIllegal ( ) , possibleParen ) ; } } while ( true ) { withItemList . push ( this . _parseWithItem ( ) ) ; if ( ! this . _consumeTokenIfType ( TokenType . Comma ) ) { break ; } if ( this . _peekToken ( ) . type === TokenType . CloseParenthesis ) { break ; } } if ( isParenthesizedWithItemList ) { if ( ! this . _consumeTokenIfType ( TokenType . CloseParenthesis ) ) { this . _addError ( Localizer . Diagnostic . expectedCloseParen ( ) , possibleParen ) ; } } const withSuite = this . _parseSuite ( this . _isInFunction ) ; const withNode = WithNode . create ( withToken , withSuite ) ; if ( asyncToken ) { withNode . isAsync = true ; withNode . asyncToken = asyncToken ; extendRange ( withNode , asyncToken ) ; } withNode . withItems = withItemList ; withItemList . forEach ( ( withItem ) => { withItem . parent = withNode ; } ) ; return withNode ; } private _parseWithItem ( ) { const expr = this . _parseTestExpression ( true ) ; const itemNode = WithItemNode . create ( expr ) ; if ( this . _consumeTokenIfKeyword ( KeywordType . As ) ) { itemNode . target = this . _parseExpression ( false ) ; itemNode . target . parent = itemNode ; extendRange ( itemNode , itemNode . target ) ; } return itemNode ; } private _parseDecorated ( ) { const decoratorList = [ ] ; while ( true ) { if ( this . _peekOperatorType ( ) === OperatorType . MatrixMultiply ) { decoratorList . push ( this . _parseDecorator ( ) ) ; } else { break ; } } const nextToken = this . _peekToken ( ) as KeywordToken ; if ( nextToken . type === TokenType . Keyword ) { if ( nextToken . keywordType === KeywordType . Async ) { this . _getNextToken ( ) ; if ( this . _peekKeywordType ( ) !== KeywordType . Def ) { this . _addError ( Localizer . Diagnostic . expectedFunctionAfterAsync ( ) , this . _peekToken ( ) ) ; } else { return this . _parseFunctionDef ( nextToken , decoratorList ) ; } } else if ( nextToken . keywordType === KeywordType . Def ) { return this . _parseFunctionDef ( undefined , decoratorList ) ; } else if ( nextToken . keywordType === KeywordType . Class ) { return this . _parseClassDef ( decoratorList ) ; } } this . _addError ( Localizer . Diagnostic . expectedAfterDecorator ( ) , this . _peekToken ( ) ) ; return ClassNode . createDummyForDecorators ( decoratorList ) ; } private _parseDecorator ( ) { const atOperator = this . _getNextToken ( ) as OperatorToken ; assert ( atOperator . operatorType === OperatorType . MatrixMultiply ) ; const expression = this . _parseTestExpression ( true ) ; if ( this . _getLanguageVersion ( ) < PythonVersion . V3_9 ) { let isSupportedExpressionForm = false ; if ( this . _isNameOrMemberAccessExpression ( expression ) ) { isSupportedExpressionForm = true ; } else if ( expression . nodeType === ParseNodeType . Call && this . _isNameOrMemberAccessExpression ( expression . leftExpression ) ) { isSupportedExpressionForm = true ; } if ( ! isSupportedExpressionForm ) { this . _addError ( Localizer . Diagnostic . expectedDecoratorExpr ( ) , expression ) ; } } const decoratorNode = DecoratorNode . create ( atOperator , expression ) ; if ( ! this . _consumeTokenIfType ( TokenType . NewLine ) ) { this . _addError ( Localizer . Diagnostic . expectedDecoratorNewline ( ) , this . _peekToken ( ) ) ; this . _consumeTokensUntilType ( [ TokenType . NewLine ] ) ; } return decoratorNode ; } private _isNameOrMemberAccessExpression ( expression ) { if ( expression . nodeType === ParseNodeType . Name ) { return true ; } else if ( expression . nodeType === ParseNodeType . MemberAccess ) { return this . _isNameOrMemberAccessExpression ( expression . leftExpression ) ; } return false ; } private _parseClassDef ( decorators ? ) { const classToken = this . _getKeywordToken ( KeywordType . Class ) ; let nameToken = this . _getTokenIfIdentifier ( ) ; if ( ! nameToken ) { this . _addError ( Localizer . Diagnostic . expectedClassName ( ) , this . _peekToken ( ) ) ; nameToken = IdentifierToken . create ( 0 , 0 , '' , undefined ) ; } let argList = [ ] ; const openParenToken = this . _peekToken ( ) ; if ( this . _consumeTokenIfType ( TokenType . OpenParenthesis ) ) { argList = this . _parseArgList ( ) ; if ( ! this . _consumeTokenIfType ( TokenType . CloseParenthesis ) ) { this . _addError ( Localizer . Diagnostic . expectedCloseParen ( ) , openParenToken ) ; } } const suite = this . _parseSuite ( false , this . _parseOptions . skipFunctionAndClassBody ) ; const classNode = ClassNode . create ( classToken , NameNode . create ( nameToken ) , suite ) ; classNode . arguments = argList ; argList . forEach ( ( arg ) => { arg . parent = classNode ; } ) ; if ( decorators ) { classNode . decorators = decorators ; if ( decorators . length > 0 ) { decorators . forEach ( ( decorator ) => { decorator . parent = classNode ; } ) ; extendRange ( classNode , decorators [ 0 ] ) ; } } return classNode ; } private _parsePassStatement ( ) { return PassNode . create ( this . _getKeywordToken ( KeywordType . Pass ) ) ; } private _parseBreakStatement ( ) { const breakToken = this . _getKeywordToken ( KeywordType . Break ) ; if ( ! this . _isInLoop ) { this . _addError ( Localizer . Diagnostic . breakOutsideLoop ( ) , breakToken ) ; } return BreakNode . create ( breakToken ) ; } private _parseContinueStatement ( ) { const continueToken = this . _getKeywordToken ( KeywordType . Continue ) ; if ( ! this . _isInLoop ) { this . _addError ( Localizer . Diagnostic . continueOutsideLoop ( ) , continueToken ) ; } else if ( this . _isInFinally ) { this . _addError ( Localizer . Diagnostic . continueInFinally ( ) , continueToken ) ; } return ContinueNode . create ( continueToken ) ; } private _parseReturnStatement ( ) { const returnToken = this . _getKeywordToken ( KeywordType . Return ) ; const returnNode = ReturnNode . create ( returnToken ) ; if ( ! this . _isInFunction ) { this . _addError ( Localizer . Diagnostic . returnOutsideFunction ( ) , returnToken ) ; } if ( ! this . _isNextTokenNeverExpression ( ) ) { const returnExpr = this . _parseTestOrStarListAsExpression ( true , ErrorExpressionCategory . MissingExpression , Localizer . Diagnostic . expectedReturnExpr ( ) ) ; this . _reportConditionalErrorForStarTupleElement ( returnExpr ) ; returnNode . returnExpression = returnExpr ; returnNode . returnExpression . parent = returnNode ; extendRange ( returnNode , returnExpr ) ; } return returnNode ; } private _parseFromStatement ( ) { const fromToken = this . _getKeywordToken ( KeywordType . From ) ; const modName = this . _parseDottedModuleName ( true ) ; const importFromNode = ImportFromNode . create ( fromToken , modName ) ; const isFutureImport = modName . leadingDots === 0 && modName . nameParts . length === 1 && modName . nameParts [ 0 ] . value === '__future__' ; const possibleInputToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfKeyword ( KeywordType . Import ) ) { this . _addError ( Localizer . Diagnostic . expectedImport ( ) , this . _peekToken ( ) ) ; if ( ! modName . hasTrailingDot ) { importFromNode . missingImportKeyword = true ; } } else { extendRange ( importFromNode , possibleInputToken ) ; const possibleStarToken = this . _peekToken ( ) ; if ( this . _consumeTokenIfOperator ( OperatorType . Multiply ) ) { extendRange ( importFromNode , possibleStarToken ) ; importFromNode . isWildcardImport = true ; importFromNode . wildcardToken = possibleStarToken ; this . _containsWildcardImport = true ; } else { const openParenToken = this . _peekToken ( ) ; const inParen = this . _consumeTokenIfType ( TokenType . OpenParenthesis ) ; let trailingCommaToken ; while ( true ) { const importName = this . _getTokenIfIdentifier ( ) ; if ( ! importName ) { break ; } trailingCommaToken = undefined ; const importFromAsNode = ImportFromAsNode . create ( NameNode . create ( importName ) ) ; if ( this . _consumeTokenIfKeyword ( KeywordType . As ) ) { const aliasName = this . _getTokenIfIdentifier ( ) ; if ( ! aliasName ) { this . _addError ( Localizer . Diagnostic . expectedImportAlias ( ) , this . _peekToken ( ) ) ; } else { importFromAsNode . alias = NameNode . create ( aliasName ) ; importFromAsNode . alias . parent = importFromAsNode ; extendRange ( importFromAsNode , aliasName ) ; } } importFromNode . imports . push ( importFromAsNode ) ; importFromAsNode . parent = importFromNode ; extendRange ( importFromNode , importFromAsNode ) ; if ( isFutureImport ) { this . _futureImportMap . set ( importName . value , true ) ; } const nextToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfType ( TokenType . Comma ) ) { break ; } trailingCommaToken = nextToken ; } if ( importFromNode . imports . length === 0 ) { this . _addError ( Localizer . Diagnostic . expectedImportSymbols ( ) , this . _peekToken ( ) ) ; } if ( inParen ) { importFromNode . usesParens = true ; const nextToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfType ( TokenType . CloseParenthesis ) ) { this . _addError ( Localizer . Diagnostic . expectedCloseParen ( ) , openParenToken ) ; } else { extendRange ( importFromNode , nextToken ) ; } } else if ( trailingCommaToken ) { this . _addError ( Localizer . Diagnostic . trailingCommaInFromImport ( ) , trailingCommaToken ) ; } } } this . _importedModules . push ( { nameNode : importFromNode . module , leadingDots : importFromNode . module . leadingDots , nameParts : importFromNode . module . nameParts . map ( ( p ) => p . value ) , importedSymbols : importFromNode . imports . map ( ( imp ) => imp . name . value ) , } ) ; let isTypingImport = false ; if ( importFromNode . module . nameParts . length === 1 ) { const firstNamePartValue = importFromNode . module . nameParts [ 0 ] . value ; if ( firstNamePartValue === 'typing' || firstNamePartValue === 'typing_extensions' ) { isTypingImport = true ; } } if ( isTypingImport ) { const typingSymbolsOfInterest = [ 'Literal' , 'TypeAlias' , 'Annotated' ] ; if ( importFromNode . isWildcardImport ) { typingSymbolsOfInterest . forEach ( ( s ) => { this . _typingSymbolAliases . set ( s , s ) ; } ) ; } else { importFromNode . imports . forEach ( ( imp ) => { if ( typingSymbolsOfInterest . some ( ( s ) => s === imp . name . value ) ) { this . _typingSymbolAliases . set ( imp . alias ?. value || imp . name . value , imp . name . value ) ; } } ) ; } } return importFromNode ; } private _parseImportStatement ( ) { const importToken = this . _getKeywordToken ( KeywordType . Import ) ; const importNode = ImportNode . create ( importToken ) ; while ( true ) { const modName = this . _parseDottedModuleName ( ) ; const importAsNode = ImportAsNode . create ( modName ) ; if ( this . _consumeTokenIfKeyword ( KeywordType . As ) ) { const aliasToken = this . _getTokenIfIdentifier ( ) ; if ( aliasToken ) { importAsNode . alias = NameNode . create ( aliasToken ) ; importAsNode . alias . parent = importAsNode ; extendRange ( importAsNode , importAsNode . alias ) ; } else { this . _addError ( Localizer . Diagnostic . expectedImportAlias ( ) , this . _peekToken ( ) ) ; } } if ( importAsNode . module . leadingDots > 0 ) { this . _addError ( Localizer . Diagnostic . relativeImportNotAllowed ( ) , importAsNode . module ) ; } importNode . list . push ( importAsNode ) ; importAsNode . parent = importNode ; this . _importedModules . push ( { nameNode : importAsNode . module , leadingDots : importAsNode . module . leadingDots , nameParts : importAsNode . module . nameParts . map ( ( p ) => p . value ) , importedSymbols : undefined , } ) ; if ( modName . nameParts . length === 1 ) { const firstNamePartValue = modName . nameParts [ 0 ] . value ; if ( firstNamePartValue === 'typing' || firstNamePartValue === 'typing_extensions' ) { this . _typingImportAliases . push ( importAsNode . alias ?. value || firstNamePartValue ) ; } } if ( ! this . _consumeTokenIfType ( TokenType . Comma ) ) { break ; } } if ( importNode . list . length > 0 ) { extendRange ( importNode , importNode . list [ importNode . list . length - 1 ] ) ; } return importNode ; } private _parseDottedModuleName ( allowJustDots = false ) { const moduleNameNode = ModuleNameNode . create ( this . _peekToken ( ) ) ; while ( true ) { const token = this . _getTokenIfType ( TokenType . Ellipsis ) ?? this . _getTokenIfType ( TokenType . Dot ) ; if ( token ) { if ( token . type === TokenType . Ellipsis ) { moduleNameNode . leadingDots += 3 ; } else { moduleNameNode . leadingDots ++ ; } extendRange ( moduleNameNode , token ) ; } else { break ; } } while ( true ) { const identifier = this . _getTokenIfIdentifier ( ) ; if ( ! identifier ) { if ( ! allowJustDots || moduleNameNode . leadingDots === 0 || moduleNameNode . nameParts . length > 0 ) { this . _addError ( Localizer . Diagnostic . expectedModuleName ( ) , this . _peekToken ( ) ) ; moduleNameNode . hasTrailingDot = true ; } break ; } const namePart = NameNode . create ( identifier ) ; moduleNameNode . nameParts . push ( namePart ) ; namePart . parent = moduleNameNode ; extendRange ( moduleNameNode , namePart ) ; const nextToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfType ( TokenType . Dot ) ) { break ; } extendRange ( moduleNameNode , nextToken ) ; } return moduleNameNode ; } private _parseGlobalStatement ( ) { const globalToken = this . _getKeywordToken ( KeywordType . Global ) ; const globalNode = GlobalNode . create ( globalToken ) ; globalNode . nameList = this . _parseNameList ( ) ; if ( globalNode . nameList . length > 0 ) { globalNode . nameList . forEach ( ( name ) => { name . parent = globalNode ; } ) ; extendRange ( globalNode , globalNode . nameList [ globalNode . nameList . length - 1 ] ) ; } return globalNode ; } private _parseNonlocalStatement ( ) { const nonlocalToken = this . _getKeywordToken ( KeywordType . Nonlocal ) ; const nonlocalNode = NonlocalNode . create ( nonlocalToken ) ; nonlocalNode . nameList = this . _parseNameList ( ) ; if ( nonlocalNode . nameList . length > 0 ) { nonlocalNode . nameList . forEach ( ( name ) => { name . parent = nonlocalNode ; } ) ; extendRange ( nonlocalNode , nonlocalNode . nameList [ nonlocalNode . nameList . length - 1 ] ) ; } return nonlocalNode ; } private _parseNameList ( ) { const nameList = [ ] ; while ( true ) { const name = this . _getTokenIfIdentifier ( ) ; if ( ! name ) { this . _addError ( Localizer . Diagnostic . expectedIdentifier ( ) , this . _peekToken ( ) ) ; break ; } nameList . push ( NameNode . create ( name ) ) ; if ( ! this . _consumeTokenIfType ( TokenType . Comma ) ) { break ; } } return nameList ; } private _parseRaiseStatement ( ) { const raiseToken = this . _getKeywordToken ( KeywordType . Raise ) ; const raiseNode = RaiseNode . create ( raiseToken ) ; if ( ! this . _isNextTokenNeverExpression ( ) ) { raiseNode . typeExpression = this . _parseTestExpression ( true ) ; raiseNode . typeExpression . parent = raiseNode ; extendRange ( raiseNode , raiseNode . typeExpression ) ; if ( this . _consumeTokenIfKeyword ( KeywordType . From ) ) { raiseNode . valueExpression = this . _parseTestExpression ( true ) ; raiseNode . valueExpression . parent = raiseNode ; extendRange ( raiseNode , raiseNode . valueExpression ) ; } else { if ( this . _consumeTokenIfType ( TokenType . Comma ) ) { raiseNode . valueExpression = this . _parseTestExpression ( true ) ; raiseNode . valueExpression . parent = raiseNode ; extendRange ( raiseNode , raiseNode . valueExpression ) ; if ( this . _consumeTokenIfType ( TokenType . Comma ) ) { raiseNode . tracebackExpression = this . _parseTestExpression ( true ) ; raiseNode . tracebackExpression . parent = raiseNode ; extendRange ( raiseNode , raiseNode . tracebackExpression ) ; } } } } return raiseNode ; } private _parseAssertStatement ( ) { const assertToken = this . _getKeywordToken ( KeywordType . Assert ) ; const expr = this . _parseTestExpression ( true ) ; const assertNode = AssertNode . create ( assertToken , expr ) ; if ( this . _consumeTokenIfType ( TokenType . Comma ) ) { const exceptionExpr = this . _parseTestExpression ( true ) ; assertNode . exceptionExpression = exceptionExpr ; assertNode . exceptionExpression . parent = assertNode ; extendRange ( assertNode , exceptionExpr ) ; } return assertNode ; } private _parseDelStatement ( ) { const delToken = this . _getKeywordToken ( KeywordType . Del ) ; const exprListResult = this . _parseExpressionList ( true ) ; if ( ! exprListResult . parseError && exprListResult . list . length === 0 ) { this . _addError ( Localizer . Diagnostic . expectedDelExpr ( ) , this . _peekToken ( ) ) ; } const delNode = DelNode . create ( delToken ) ; delNode . expressions = exprListResult . list ; if ( delNode . expressions . length > 0 ) { delNode . expressions . forEach ( ( expr ) => { expr . parent = delNode ; } ) ; extendRange ( delNode , delNode . expressions [ delNode . expressions . length - 1 ] ) ; } return delNode ; } private _parseYieldExpression ( ) { const yieldToken = this . _getKeywordToken ( KeywordType . Yield ) ; const nextToken = this . _peekToken ( ) ; if ( this . _consumeTokenIfKeyword ( KeywordType . From ) ) { if ( this . _getLanguageVersion ( ) < PythonVersion . V3_3 ) { this . _addError ( Localizer . Diagnostic . yieldFromIllegal ( ) , nextToken ) ; } return YieldFromNode . create ( yieldToken , this . _parseTestExpression ( true ) ) ; } let exprList ; if ( ! this . _isNextTokenNeverExpression ( ) ) { exprList = this . _parseTestOrStarListAsExpression ( true , ErrorExpressionCategory . MissingExpression , Localizer . Diagnostic . expectedYieldExpr ( ) ) ; this . _reportConditionalErrorForStarTupleElement ( exprList ) ; } return YieldNode . create ( yieldToken , exprList ) ; } private _tryParseYieldExpression ( ) { if ( this . _peekKeywordType ( ) !== KeywordType . Yield ) { return undefined ; } return this . _parseYieldExpression ( ) ; } private _parseSimpleStatement ( ) { const statement = StatementListNode . create ( this . _peekToken ( ) ) ; while ( true ) { if ( this . _peekTokenType ( ) === TokenType . Invalid ) { const invalidToken = this . _getNextToken ( ) ; const text = this . _fileContents ! . substr ( invalidToken . start , invalidToken . length ) ; const firstCharCode = text . charCodeAt ( 0 ) ; this . _addError ( Localizer . Diagnostic . invalidTokenChars ( ) . format ( { text : `\\u${ firstCharCode . toString ( 16 ) }` } ) , invalidToken ) ; this . _consumeTokensUntilType ( [ TokenType . NewLine ] ) ; break ; } const smallStatement = this . _parseSmallStatement ( ) ; statement . statements . push ( smallStatement ) ; smallStatement . parent = statement ; extendRange ( statement , smallStatement ) ; if ( smallStatement . nodeType === ParseNodeType . Error ) { break ; } if ( ! this . _consumeTokenIfType ( TokenType . Semicolon ) ) { break ; } const nextTokenType = this . _peekTokenType ( ) ; if ( nextTokenType === TokenType . NewLine || nextTokenType === TokenType . EndOfStream ) { break ; } } if ( ! this . _consumeTokenIfType ( TokenType . NewLine ) ) { this . _addError ( Localizer . Diagnostic . expectedNewlineOrSemicolon ( ) , this . _peekToken ( ) ) ; } return statement ; } private _parseSmallStatement ( ) { switch ( this . _peekKeywordType ( ) ) { case KeywordType . Pass : return this . _parsePassStatement ( ) ; case KeywordType . Break : return this . _parseBreakStatement ( ) ; case KeywordType . Continue : return this . _parseContinueStatement ( ) ; case KeywordType . Return : return this . _parseReturnStatement ( ) ; case KeywordType . From : return this . _parseFromStatement ( ) ; case KeywordType . Import : return this . _parseImportStatement ( ) ; case KeywordType . Global : return this . _parseGlobalStatement ( ) ; case KeywordType . Nonlocal : return this . _parseNonlocalStatement ( ) ; case KeywordType . Raise : return this . _parseRaiseStatement ( ) ; case KeywordType . Assert : return this . _parseAssertStatement ( ) ; case KeywordType . Del : return this . _parseDelStatement ( ) ; case KeywordType . Yield : return this . _parseYieldExpression ( ) ; } return this . _parseExpressionStatement ( ) ; } private _makeExpressionOrTuple ( exprListResult , enclosedInParens ) { if ( exprListResult . list . length === 1 && ! exprListResult . trailingComma ) { if ( exprListResult . list [ 0 ] . nodeType === ParseNodeType . Unpack ) { this . _addError ( Localizer . Diagnostic . unpackNotAllowed ( ) , exprListResult . list [ 0 ] ) ; } return exprListResult . list [ 0 ] ; } const tupleStartRange = exprListResult . list . length > 0 ? exprListResult . list [ 0 ] : this . _peekToken ( - 1 ) ; const tupleNode = TupleNode . create ( tupleStartRange , enclosedInParens ) ; tupleNode . expressions = exprListResult . list ; if ( exprListResult . list . length > 0 ) { exprListResult . list . forEach ( ( expr ) => { expr . parent = tupleNode ; } ) ; extendRange ( tupleNode , exprListResult . list [ exprListResult . list . length - 1 ] ) ; } return tupleNode ; } private _parseExpressionListAsPossibleTuple ( errorCategory , errorString , errorToken ) { if ( this . _isNextTokenNeverExpression ( ) ) { this . _addError ( errorString , errorToken ) ; return ErrorNode . create ( errorToken , errorCategory ) ; } const exprListResult = this . _parseExpressionList ( true ) ; if ( exprListResult . parseError ) { return exprListResult . parseError ; } return this . _makeExpressionOrTuple ( exprListResult , false ) ; } private _parseTestListAsExpression ( errorCategory , errorString ) { if ( this . _isNextTokenNeverExpression ( ) ) { return this . _handleExpressionParseError ( errorCategory , errorString ) ; } const exprListResult = this . _parseTestExpressionList ( ) ; if ( exprListResult . parseError ) { return exprListResult . parseError ; } return this . _makeExpressionOrTuple ( exprListResult , false ) ; } private _parseTestOrStarListAsExpression ( allowAssignmentExpression , errorCategory , errorString ) { if ( this . _isNextTokenNeverExpression ( ) ) { return this . _handleExpressionParseError ( errorCategory , errorString ) ; } const exprListResult = this . _parseTestOrStarExpressionList ( allowAssignmentExpression ) ; if ( exprListResult . parseError ) { return exprListResult . parseError ; } return this . _makeExpressionOrTuple ( exprListResult , false ) ; } private _parseExpressionList ( allowStar ) { return this . _parseExpressionListGeneric ( ( ) => this . _parseExpression ( allowStar ) ) ; } private _parseTestExpressionList ( ) { return this . _parseExpressionListGeneric ( ( ) => this . _parseTestExpression ( false ) ) ; } private _parseTestOrStarExpressionList ( allowAssignmentExpression ) { const exprListResult = this . _parseExpressionListGeneric ( ( ) => this . _parseTestOrStarExpression ( allowAssignmentExpression ) ) ; if ( ! exprListResult . parseError ) { let sawStar = false ; for ( const expr of exprListResult . list ) { if ( expr . nodeType === ParseNodeType . Unpack ) { if ( sawStar ) { this . _addError ( Localizer . Diagnostic . duplicateUnpack ( ) , expr ) ; break ; } sawStar = true ; } } } return exprListResult ; } private _parseExpression ( allowUnpack ) { const startToken = this . _peekToken ( ) ; if ( allowUnpack && this . _consumeTokenIfOperator ( OperatorType . Multiply ) ) { return UnpackNode . create ( startToken , this . _parseExpression ( false ) ) ; } return this . _parseBitwiseOrExpression ( ) ; } private _parseTestOrStarExpression ( allowAssignmentExpression ) { if ( this . _peekOperatorType ( ) === OperatorType . Multiply ) { return this . _parseExpression ( true ) ; } return this . _parseTestExpression ( allowAssignmentExpression ) ; } private _parseTestExpression ( allowAssignmentExpression ) { if ( this . _peekKeywordType ( ) === KeywordType . Lambda ) { return this . _parseLambdaExpression ( ) ; } const ifExpr = allowAssignmentExpression ? this . _parseAssignmentExpression ( ) : this . _parseOrTest ( ) ; if ( ifExpr . nodeType === ParseNodeType . Error ) { return ifExpr ; } if ( ! this . _consumeTokenIfKeyword ( KeywordType . If ) ) { return ifExpr ; } const testExpr = this . _parseOrTest ( ) ; if ( testExpr . nodeType === ParseNodeType . Error ) { return testExpr ; } if ( ! this . _consumeTokenIfKeyword ( KeywordType . Else ) ) { return this . _handleExpressionParseError ( ErrorExpressionCategory . MissingElse , Localizer . Diagnostic . expectedElse ( ) ) ; } const elseExpr = this . _parseTestExpression ( true ) ; if ( elseExpr . nodeType === ParseNodeType . Error ) { return elseExpr ; } return TernaryNode . create ( ifExpr , testExpr , elseExpr ) ; } private _parseAssignmentExpression ( ) { const leftExpr = this . _parseOrTest ( ) ; if ( leftExpr . nodeType === ParseNodeType . Error ) { return leftExpr ; } if ( leftExpr . nodeType !== ParseNodeType . Name ) { return leftExpr ; } const walrusToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfOperator ( OperatorType . Walrus ) ) { return leftExpr ; } if ( ! this . _assignmentExpressionsAllowed || this . _isParsingTypeAnnotation ) { this . _addError ( Localizer . Diagnostic . walrusNotAllowed ( ) , walrusToken ) ; } if ( this . _getLanguageVersion ( ) < PythonVersion . V3_8 ) { this . _addError ( Localizer . Diagnostic . walrusIllegal ( ) , walrusToken ) ; } const rightExpr = this . _parseTestExpression ( false ) ; return AssignmentExpressionNode . create ( leftExpr , rightExpr ) ; } private _parseOrTest ( ) { if ( this . _peekKeywordType ( ) === KeywordType . Async || this . _peekTokenType ( ) === TokenType . OpenParenthesis ) { const possibleArrowCall = this . _parseArrowCall ( ) ; if ( possibleArrowCall ) { return possibleArrowCall ; } } let leftExpr = this . _parseAndTest ( ) ; if ( leftExpr . nodeType === ParseNodeType . Error ) { return leftExpr ; } while ( true ) { const peekToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfKeyword ( KeywordType . Or ) ) { break ; } const rightExpr = this . _parseAndTest ( ) ; leftExpr = BinaryOperationNode . create ( leftExpr , rightExpr , peekToken , OperatorType . Or ) ; } return leftExpr ; } private _parseAndTest ( ) { let leftExpr = this . _parseNotTest ( ) ; if ( leftExpr . nodeType === ParseNodeType . Error ) { return leftExpr ; } while ( true ) { const peekToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfKeyword ( KeywordType . And ) ) { break ; } const rightExpr = this . _parseNotTest ( ) ; leftExpr = BinaryOperationNode . create ( leftExpr , rightExpr , peekToken , OperatorType . And ) ; } return leftExpr ; } private _parseNotTest ( ) { const notToken = this . _peekToken ( ) ; if ( this . _consumeTokenIfKeyword ( KeywordType . Not ) ) { const notExpr = this . _parseNotTest ( ) ; return UnaryOperationNode . create ( notToken , notExpr , OperatorType . Not ) ; } return this . _parseComparison ( ) ; } private _parseComparison ( ) { let leftExpr = this . _parseBitwiseOrExpression ( ) ; if ( leftExpr . nodeType === ParseNodeType . Error ) { return leftExpr ; } while ( true ) { let comparisonOperator ; const peekToken = this . _peekToken ( ) ; if ( Tokenizer . isOperatorComparison ( this . _peekOperatorType ( ) ) ) { comparisonOperator = this . _peekOperatorType ( ) ; if ( comparisonOperator === OperatorType . LessOrGreaterThan ) { this . _addError ( Localizer . Diagnostic . operatorLessOrGreaterDeprecated ( ) , peekToken ) ; comparisonOperator = OperatorType . NotEquals ; } this . _getNextToken ( ) ; } else if ( this . _consumeTokenIfKeyword ( KeywordType . In ) ) { comparisonOperator = OperatorType . In ; } else if ( this . _consumeTokenIfKeyword ( KeywordType . Is ) ) { if ( this . _consumeTokenIfKeyword ( KeywordType . Not ) ) { comparisonOperator = OperatorType . IsNot ; } else { comparisonOperator = OperatorType . Is ; } } else if ( this . _peekKeywordType ( ) === KeywordType . Not ) { const tokenAfterNot = this . _peekToken ( 1 ) ; if ( tokenAfterNot . type === TokenType . Keyword && ( tokenAfterNot as KeywordToken ) . keywordType === KeywordType . In ) { this . _getNextToken ( ) ; this . _getNextToken ( ) ; comparisonOperator = OperatorType . NotIn ; } } if ( comparisonOperator === undefined ) { break ; } const rightExpr = this . _parseComparison ( ) ; leftExpr = BinaryOperationNode . create ( leftExpr , rightExpr , peekToken , comparisonOperator ) ; } return leftExpr ; } private _parseBitwiseOrExpression ( ) { let leftExpr = this . _parseBitwiseXorExpression ( ) ; if ( leftExpr . nodeType === ParseNodeType . Error ) { return leftExpr ; } while ( true ) { const peekToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfOperator ( OperatorType . BitwiseOr ) ) { break ; } const rightExpr = this . _parseBitwiseXorExpression ( ) ; leftExpr = BinaryOperationNode . create ( leftExpr , rightExpr , peekToken , OperatorType . BitwiseOr ) ; } return leftExpr ; } private _parseBitwiseXorExpression ( ) { let leftExpr = this . _parseBitwiseAndExpression ( ) ; if ( leftExpr . nodeType === ParseNodeType . Error ) { return leftExpr ; } while ( true ) { const peekToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfOperator ( OperatorType . BitwiseXor ) ) { break ; } const rightExpr = this . _parseBitwiseAndExpression ( ) ; leftExpr = BinaryOperationNode . create ( leftExpr , rightExpr , peekToken , OperatorType . BitwiseXor ) ; } return leftExpr ; } private _parseBitwiseAndExpression ( ) { let leftExpr = this . _parseShiftExpression ( ) ; if ( leftExpr . nodeType === ParseNodeType . Error ) { return leftExpr ; } while ( true ) { const peekToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfOperator ( OperatorType . BitwiseAnd ) ) { break ; } const rightExpr = this . _parseShiftExpression ( ) ; leftExpr = BinaryOperationNode . create ( leftExpr , rightExpr , peekToken , OperatorType . BitwiseAnd ) ; } return leftExpr ; } private _parseShiftExpression ( ) { let leftExpr = this . _parseArithmeticExpression ( ) ; if ( leftExpr . nodeType === ParseNodeType . Error ) { return leftExpr ; } let peekToken = this . _peekToken ( ) ; let nextOperator = this . _peekOperatorType ( ) ; while ( nextOperator === OperatorType . LeftShift || nextOperator === OperatorType . RightShift ) { this . _getNextToken ( ) ; const rightExpr = this . _parseArithmeticExpression ( ) ; leftExpr = BinaryOperationNode . create ( leftExpr , rightExpr , peekToken , nextOperator ) ; peekToken = this . _peekToken ( ) ; nextOperator = this . _peekOperatorType ( ) ; } return leftExpr ; } private _parseArithmeticExpression ( ) { let leftExpr = this . _parseArithmeticTerm ( ) ; if ( leftExpr . nodeType === ParseNodeType . Error ) { return leftExpr ; } let peekToken = this . _peekToken ( ) ; let nextOperator = this . _peekOperatorType ( ) ; while ( nextOperator === OperatorType . Add || nextOperator === OperatorType . Subtract ) { this . _getNextToken ( ) ; const rightExpr = this . _parseArithmeticTerm ( ) ; if ( rightExpr . nodeType === ParseNodeType . Error ) { return rightExpr ; } leftExpr = BinaryOperationNode . create ( leftExpr , rightExpr , peekToken , nextOperator ) ; peekToken = this . _peekToken ( ) ; nextOperator = this . _peekOperatorType ( ) ; } return leftExpr ; } private _parseArithmeticTerm ( ) { let leftExpr = this . _parseArithmeticFactor ( ) ; if ( leftExpr . nodeType === ParseNodeType . Error ) { return leftExpr ; } let peekToken = this . _peekToken ( ) ; let nextOperator = this . _peekOperatorType ( ) ; while ( nextOperator === OperatorType . Multiply || nextOperator === OperatorType . MatrixMultiply || nextOperator === OperatorType . Divide || nextOperator === OperatorType . Mod || nextOperator === OperatorType . FloorDivide ) { this . _getNextToken ( ) ; const rightExpr = this . _parseArithmeticFactor ( ) ; leftExpr = BinaryOperationNode . create ( leftExpr , rightExpr , peekToken , nextOperator ) ; peekToken = this . _peekToken ( ) ; nextOperator = this . _peekOperatorType ( ) ; } return leftExpr ; } private _parseArithmeticFactor ( ) { const nextToken = this . _peekToken ( ) ; const nextOperator = this . _peekOperatorType ( ) ; if ( nextOperator === OperatorType . Add || nextOperator === OperatorType . Subtract || nextOperator === OperatorType . BitwiseInvert ) { this . _getNextToken ( ) ; const expression = this . _parseArithmeticFactor ( ) ; return UnaryOperationNode . create ( nextToken , expression , nextOperator ) ; } const leftExpr = this . _parseAtomExpression ( ) ; if ( leftExpr . nodeType === ParseNodeType . Error ) { return leftExpr ; } const peekToken = this . _peekToken ( ) ; if ( this . _consumeTokenIfOperator ( OperatorType . Power ) ) { const rightExpr = this . _parseArithmeticFactor ( ) ; return BinaryOperationNode . create ( leftExpr , rightExpr , peekToken , OperatorType . Power ) ; } return leftExpr ; } private _isTypingAnnotation ( typeAnnotation , name ) { if ( typeAnnotation . nodeType === ParseNodeType . Name ) { const alias = this . _typingSymbolAliases . get ( typeAnnotation . value ) ; if ( alias === name ) { return true ; } } else if ( typeAnnotation . nodeType === ParseNodeType . MemberAccess ) { if ( typeAnnotation . leftExpression . nodeType === ParseNodeType . Name && typeAnnotation . memberName . value === name ) { const baseName = typeAnnotation . leftExpression . value ; return this . _typingImportAliases . some ( ( alias ) => alias === baseName ) ; } } return false ; } private _parseAtomExpression ( ) { let awaitToken ; if ( this . _peekKeywordType ( ) === KeywordType . Await && ! this . _isParsingTypeAnnotation ) { awaitToken = this . _getKeywordToken ( KeywordType . Await ) ; if ( this . _getLanguageVersion ( ) < PythonVersion . V3_5 ) { this . _addError ( Localizer . Diagnostic . awaitIllegal ( ) , awaitToken ) ; } } let atomExpression = this . _parseAtom ( ) ; if ( atomExpression . nodeType === ParseNodeType . Error ) { return atomExpression ; } while ( true ) { const startOfTrailerToken = this . _peekToken ( ) ; if ( this . _consumeTokenIfType ( TokenType . OpenParenthesis ) ) { const wasParsingTypeAnnotation = this . _isParsingTypeAnnotation ; this . _isParsingTypeAnnotation = false ; const argList = this . _parseArgList ( ) ; const callNode = CallNode . create ( atomExpression ) ; callNode . arguments = argList ; if ( argList . length > 0 ) { argList . forEach ( ( arg ) => { arg . parent = callNode ; } ) ; extendRange ( callNode , argList [ argList . length - 1 ] ) ; } const nextToken = this . _peekToken ( ) ; let isArgListTerminated = false ; if ( ! this . _consumeTokenIfType ( TokenType . CloseParenthesis ) ) { this . _addError ( Localizer . Diagnostic . expectedCloseParen ( ) , startOfTrailerToken ) ; this . _consumeTokensUntilType ( [ TokenType . NewLine ] ) ; extendRange ( callNode , this . _peekToken ( ) ) ; } else { extendRange ( callNode , nextToken ) ; isArgListTerminated = true ; } this . _isParsingTypeAnnotation = wasParsingTypeAnnotation ; if ( this . _isParsingTypeAnnotation ) { const diag = new DiagnosticAddendum ( ) ; if ( atomExpression . nodeType === ParseNodeType . Name && atomExpression . value === 'type' ) { diag . addMessage ( Localizer . DiagnosticAddendum . useTypeInstead ( ) ) ; this . _addError ( Localizer . Diagnostic . typeCallNotAllowed ( ) + diag . getString ( ) , callNode ) ; } } atomExpression = callNode ; if ( ! isArgListTerminated ) { break ; } } else if ( this . _consumeTokenIfType ( TokenType . OpenBracket ) ) { const wasParsingIndexTrailer = this . _isParsingIndexTrailer ; const wasParsingTypeAnnotation = this . _isParsingTypeAnnotation ; if ( this . _isTypingAnnotation ( atomExpression , 'Literal' ) || this . _isTypingAnnotation ( atomExpression , 'Annotated' ) ) { this . _isParsingTypeAnnotation = false ; } this . _isParsingIndexTrailer = true ; const subscriptList = this . _parseSubscriptList ( ) ; this . _isParsingTypeAnnotation = wasParsingTypeAnnotation ; this . _isParsingIndexTrailer = wasParsingIndexTrailer ; const closingToken = this . _peekToken ( ) ; const indexNode = IndexNode . create ( atomExpression , subscriptList . list , subscriptList . trailingComma , closingToken ) ; extendRange ( indexNode , indexNode ) ; if ( ! this . _consumeTokenIfType ( TokenType . CloseBracket ) ) { this . _handleExpressionParseError ( ErrorExpressionCategory . MissingIndexCloseBracket , Localizer . Diagnostic . expectedCloseBracket ( ) , startOfTrailerToken , indexNode ) ; } atomExpression = indexNode ; } else if ( this . _consumeTokenIfType ( TokenType . Dot ) ) { const memberName = this . _getTokenIfIdentifier ( ) ; if ( ! memberName ) { return this . _handleExpressionParseError ( ErrorExpressionCategory . MissingMemberAccessName , Localizer . Diagnostic . expectedMemberName ( ) , startOfTrailerToken , atomExpression ) ; } atomExpression = MemberAccessNode . create ( atomExpression , NameNode . create ( memberName ) ) ; } else { break ; } } if ( awaitToken ) { return AwaitNode . create ( awaitToken , atomExpression ) ; } return atomExpression ; } private _parseSubscriptList ( ) { const argList = [ ] ; let sawKeywordArg = false ; let trailingComma = false ; while ( true ) { const firstToken = this . _peekToken ( ) ; if ( firstToken . type !== TokenType . Colon && this . _isNextTokenNeverExpression ( ) ) { break ; } let argType = ArgumentCategory . Simple ; if ( this . _consumeTokenIfOperator ( OperatorType . Multiply ) ) { argType = ArgumentCategory . UnpackedList ; } else if ( this . _consumeTokenIfOperator ( OperatorType . Power ) ) { argType = ArgumentCategory . UnpackedDictionary ; } let valueExpr = this . _parsePossibleSlice ( ) ; let nameIdentifier ; if ( argType === ArgumentCategory . Simple ) { if ( this . _consumeTokenIfOperator ( OperatorType . Assign ) ) { const nameExpr = valueExpr ; valueExpr = this . _parsePossibleSlice ( ) ; if ( nameExpr . nodeType === ParseNodeType . Name ) { nameIdentifier = nameExpr . token ; } else { this . _addError ( Localizer . Diagnostic . expectedParamName ( ) , nameExpr ) ; } } } const argNode = ArgumentNode . create ( firstToken , valueExpr , argType ) ; if ( nameIdentifier ) { argNode . name = NameNode . create ( nameIdentifier ) ; argNode . name . parent = argNode ; } if ( argNode . name ) { sawKeywordArg = true ; } else if ( sawKeywordArg && argNode . argumentCategory === ArgumentCategory . Simple ) { this . _addError ( Localizer . Diagnostic . positionArgAfterNamedArg ( ) , argNode ) ; } argList . push ( argNode ) ; if ( supportPEP637 ) { if ( ! this . _parseOptions . isStubFile && this . _getLanguageVersion ( ) < PythonVersion . V3_10 ) { if ( argNode . name ) { this . _addError ( Localizer . Diagnostic . keywordSubscriptIllegal ( ) , argNode . name ) ; } if ( argType !== ArgumentCategory . Simple ) { this . _addError ( Localizer . Diagnostic . unpackedSubscriptIllegal ( ) , argNode ) ; } } } else { if ( argNode . name ) { this . _addError ( Localizer . Diagnostic . keywordSubscriptIllegal ( ) , argNode . name ) ; } if ( argType !== ArgumentCategory . Simple ) { this . _addError ( Localizer . Diagnostic . unpackedSubscriptIllegal ( ) , argNode ) ; } } if ( ! this . _consumeTokenIfType ( TokenType . Comma ) ) { trailingComma = false ; break ; } trailingComma = true ; } if ( argList . length === 0 ) { const errorNode = this . _handleExpressionParseError ( ErrorExpressionCategory . MissingIndexOrSlice , Localizer . Diagnostic . expectedSliceIndex ( ) , undefined , undefined , [ TokenType . CloseBracket ] ) ; argList . push ( ArgumentNode . create ( this . _peekToken ( ) , errorNode , ArgumentCategory . Simple ) ) ; } return { list : argList , trailingComma , } ; } private _parsePossibleSlice ( ) { const firstToken = this . _peekToken ( ) ; const sliceExpressions = [ undefined , undefined , undefined ] ; let sliceIndex = 0 ; let sawColon = false ; while ( true ) { const nextTokenType = this . _peekTokenType ( ) ; if ( nextTokenType === TokenType . CloseBracket || nextTokenType === TokenType . Comma ) { break ; } if ( nextTokenType !== TokenType . Colon ) { sliceExpressions [ sliceIndex ] = this . _parseTestExpression ( false ) ; } sliceIndex ++ ; if ( sliceIndex >= 3 || ! this . _consumeTokenIfType ( TokenType . Colon ) ) { break ; } sawColon = true ; } if ( ! sawColon ) { if ( sliceExpressions [ 0 ] ) { return sliceExpressions [ 0 ] ; } return ErrorNode . create ( this . _peekToken ( ) , ErrorExpressionCategory . MissingIndexOrSlice ) ; } const sliceNode = SliceNode . create ( firstToken ) ; sliceNode . startValue = sliceExpressions [ 0 ] ; if ( sliceNode . startValue ) { sliceNode . startValue . parent = sliceNode ; } sliceNode . endValue = sliceExpressions [ 1 ] ; if ( sliceNode . endValue ) { sliceNode . endValue . parent = sliceNode ; } sliceNode . stepValue = sliceExpressions [ 2 ] ; if ( sliceNode . stepValue ) { sliceNode . stepValue . parent = sliceNode ; } const extension = sliceExpressions [ 2 ] || sliceExpressions [ 1 ] || sliceExpressions [ 0 ] ; if ( extension ) { extendRange ( sliceNode , extension ) ; } return sliceNode ; } private _parseArgList ( ) { const argList = [ ] ; let sawKeywordArg = false ; while ( true ) { const nextTokenType = this . _peekTokenType ( ) ; if ( nextTokenType === TokenType . CloseParenthesis || nextTokenType === TokenType . NewLine || nextTokenType === TokenType . EndOfStream ) { break ; } const arg = this . _parseArgument ( ) ; if ( arg . name ) { sawKeywordArg = true ; } else if ( sawKeywordArg && arg . argumentCategory === ArgumentCategory . Simple ) { this . _addError ( Localizer . Diagnostic . positionArgAfterNamedArg ( ) , arg ) ; } argList . push ( arg ) ; if ( ! this . _consumeTokenIfType ( TokenType . Comma ) ) { break ; } } return argList ; } private _parseArgument ( ) { const firstToken = this . _peekToken ( ) ; let argType = ArgumentCategory . Simple ; if ( this . _consumeTokenIfOperator ( OperatorType . Multiply ) ) { argType = ArgumentCategory . UnpackedList ; } else if ( this . _consumeTokenIfOperator ( OperatorType . Power ) ) { argType = ArgumentCategory . UnpackedDictionary ; } let valueExpr = this . _parseTestExpression ( true ) ; let nameIdentifier ; if ( argType === ArgumentCategory . Simple ) { if ( this . _consumeTokenIfOperator ( OperatorType . Assign ) ) { const nameExpr = valueExpr ; valueExpr = this . _parseTestExpression ( false ) ; if ( nameExpr . nodeType === ParseNodeType . Name ) { nameIdentifier = nameExpr . token ; } else { this . _addError ( Localizer . Diagnostic . expectedParamName ( ) , nameExpr ) ; } } else { const listComp = this . _tryParseListComprehension ( valueExpr ) ; if ( listComp ) { valueExpr = listComp ; } } } const argNode = ArgumentNode . create ( firstToken , valueExpr , argType ) ; if ( nameIdentifier ) { argNode . name = NameNode . create ( nameIdentifier ) ; argNode . name . parent = argNode ; } return argNode ; } private _parseAtom ( ) { const nextToken = this . _peekToken ( ) ; if ( nextToken . type === TokenType . Ellipsis ) { return EllipsisNode . create ( this . _getNextToken ( ) ) ; } if ( nextToken . type === TokenType . Number ) { return NumberNode . create ( this . _getNextToken ( ) as NumberToken ) ; } if ( nextToken . type === TokenType . Identifier ) { return NameNode . create ( this . _getNextToken ( ) as IdentifierToken ) ; } if ( nextToken . type === TokenType . String ) { return this . _parseStringList ( ) ; } if ( nextToken . type === TokenType . Backtick ) { this . _getNextToken ( ) ; this . _addError ( Localizer . Diagnostic . backticksIllegal ( ) , nextToken ) ; const expressionNode = this . _parseTestListAsExpression ( ErrorExpressionCategory . MissingExpression , Localizer . Diagnostic . expectedExpr ( ) ) ; this . _consumeTokenIfType ( TokenType . Backtick ) ; return expressionNode ; } if ( nextToken . type === TokenType . OpenParenthesis ) { const possibleArrowCall = this . _parseArrowCall ( ) ; if ( possibleArrowCall ) { this . _addError ( Localizer . Diagnostic . arrowCallableNeedsParen ( ) , possibleArrowCall ) ; return possibleArrowCall ; } const possibleTupleNode = this . _parseTupleAtom ( ) ; if ( possibleTupleNode . nodeType === ParseNodeType . Tuple && this . _isParsingTypeAnnotation && ! this . _isParsingIndexTrailer ) { const diag = new DiagnosticAddendum ( ) ; diag . addMessage ( Localizer . DiagnosticAddendum . useTupleInstead ( ) ) ; this . _addError ( Localizer . Diagnostic . tupleInAnnotation ( ) + diag . getString ( ) , possibleTupleNode ) ; } if ( possibleTupleNode . nodeType === ParseNodeType . BinaryOperation ) { possibleTupleNode . parenthesized = true ; } if ( possibleTupleNode . nodeType === ParseNodeType . StringList ) { possibleTupleNode . isParenthesized = true ; } return possibleTupleNode ; } else if ( nextToken . type === TokenType . OpenBracket ) { const listNode = this . _parseListAtom ( ) ; if ( this . _isParsingTypeAnnotation && ! this . _isParsingIndexTrailer ) { const diag = new DiagnosticAddendum ( ) ; diag . addMessage ( Localizer . DiagnosticAddendum . useListInstead ( ) ) ; this . _addError ( Localizer . Diagnostic . listInAnnotation ( ) + diag . getString ( ) , listNode ) ; } return listNode ; } else if ( nextToken . type === TokenType . OpenCurlyBrace ) { const dictNode = this . _parseDictionaryOrSetAtom ( ) ; if ( this . _isParsingTypeAnnotation ) { const diag = new DiagnosticAddendum ( ) ; diag . addMessage ( Localizer . DiagnosticAddendum . useDictInstead ( ) ) ; this . _addError ( Localizer . Diagnostic . dictInAnnotation ( ) + diag . getString ( ) , dictNode ) ; } return dictNode ; } if ( nextToken . type === TokenType . Keyword ) { const keywordToken = nextToken as KeywordToken ; if ( keywordToken . keywordType === KeywordType . False || keywordToken . keywordType === KeywordType . True || keywordToken . keywordType === KeywordType . Debug || keywordToken . keywordType === KeywordType . None ) { return ConstantNode . create ( this . _getNextToken ( ) as KeywordToken ) ; } if ( keywordToken . keywordType === KeywordType . Async ) { const possibleArrowCall = this . _parseArrowCall ( ) ; if ( possibleArrowCall ) { this . _addError ( Localizer . Diagnostic . arrowCallableNeedsParen ( ) , possibleArrowCall ) ; return possibleArrowCall ; } } const keywordAsIdentifier = this . _getTokenIfIdentifier ( ) ; if ( keywordAsIdentifier ) { return NameNode . create ( keywordAsIdentifier ) ; } } return this . _handleExpressionParseError ( ErrorExpressionCategory . MissingExpression , Localizer . Diagnostic . expectedExpr ( ) ) ; } private _handleExpressionParseError ( category , errorMsg , targetToken ? , childNode ? , additionalStopTokens ? ) { this . _addError ( errorMsg , targetToken ?? this . _peekToken ( ) ) ; const expr = ErrorNode . create ( this . _peekToken ( ) , category , childNode ) ; const stopTokens = [ TokenType . NewLine ] ; if ( additionalStopTokens ) { stopTokens . push ( ... additionalStopTokens ) ; } this . _consumeTokensUntilType ( stopTokens ) ; return expr ; } private _parseLambdaExpression ( allowConditional = true ) { const lambdaToken = this . _getKeywordToken ( KeywordType . Lambda ) ; const argList = this . _parseVarArgsList ( TokenType . Colon , false ) ; if ( ! this . _consumeTokenIfType ( TokenType . Colon ) ) { this . _addError ( Localizer . Diagnostic . expectedColon ( ) , this . _peekToken ( ) ) ; } let testExpr ; if ( allowConditional ) { testExpr = this . _parseTestExpression ( false ) ; } else { testExpr = this . _tryParseLambdaExpression ( false ) || this . _parseOrTest ( ) ; } const lambdaNode = LambdaNode . create ( lambdaToken , testExpr ) ; lambdaNode . parameters = argList ; argList . forEach ( ( arg ) => { arg . parent = lambdaNode ; } ) ; return lambdaNode ; } private _tryParseLambdaExpression ( allowConditional = true ) { if ( this . _peekKeywordType ( ) !== KeywordType . Lambda ) { return undefined ; } return this . _parseLambdaExpression ( allowConditional ) ; } private _parseArrowCall ( ) { let smellsLikeArrowCall = false ; this . _suppressErrors ( ( ) => { const curTokenIndex = this . _tokenIndex ; this . _consumeTokenIfKeyword ( KeywordType . Async ) ; if ( this . _consumeTokenIfType ( TokenType . OpenParenthesis ) ) { this . _parseArrowCallableParameterList ( ) ; if ( this . _consumeTokenIfType ( TokenType . CloseParenthesis ) ) { if ( this . _consumeTokenIfType ( TokenType . Arrow ) ) { smellsLikeArrowCall = true ; } } } this . _tokenIndex = curTokenIndex ; } ) ; if ( ! smellsLikeArrowCall ) { return undefined ; } let asyncToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfKeyword ( KeywordType . Async ) ) { asyncToken = undefined ; } const openParenToken = this . _peekToken ( ) ; if ( openParenToken . type !== TokenType . OpenParenthesis ) { return undefined ; } this . _getNextToken ( ) ; const arrowCallableParams = this . _parseArrowCallableParameterList ( ) ; this . _getNextToken ( ) ; this . _getNextToken ( ) ; const returnTypeAnnotation = this . _parseTypeAnnotation ( ) ; return ArrowCallableNode . create ( openParenToken , arrowCallableParams , returnTypeAnnotation , asyncToken ) ; } private _parseArrowCallableParameterList ( ) { const paramList = [ ] ; while ( true ) { if ( this . _peekTokenType ( ) === TokenType . CloseParenthesis ) { break ; } const param = this . _parseArrowCallableParameter ( ) ; if ( ! param ) { this . _consumeTokensUntilType ( [ TokenType . CloseParenthesis ] ) ; break ; } paramList . push ( param ) ; const foundComma = this . _consumeTokenIfType ( TokenType . Comma ) ; if ( ! foundComma ) { break ; } } return paramList ; } private _parseArrowCallableParameter ( ) { const nextToken = this . _peekToken ( ) ; let category = ParameterCategory . Simple ; let starToken ; if ( this . _consumeTokenIfOperator ( OperatorType . Multiply ) ) { category = ParameterCategory . VarArgList ; starToken = nextToken ; } else if ( this . _consumeTokenIfOperator ( OperatorType . Power ) ) { category = ParameterCategory . VarArgDictionary ; starToken = nextToken ; } const typeAnnotation = this . _parseTypeAnnotation ( ) ; return { category , typeAnnotation , starToken } ; } private _parseTupleAtom ( ) { const startParen = this . _getNextToken ( ) ; assert ( startParen . type === TokenType . OpenParenthesis ) ; const yieldExpr = this . _tryParseYieldExpression ( ) ; if ( yieldExpr ) { if ( this . _peekTokenType ( ) !== TokenType . CloseParenthesis ) { return this . _handleExpressionParseError ( ErrorExpressionCategory . MissingTupleCloseParen , Localizer . Diagnostic . expectedCloseParen ( ) , startParen , yieldExpr ) ; } else { extendRange ( yieldExpr , this . _getNextToken ( ) ) ; } return yieldExpr ; } const exprListResult = this . _parseTestListWithComprehension ( ) ; const tupleOrExpression = this . _makeExpressionOrTuple ( exprListResult , true ) ; const isExpression = exprListResult . list . length === 1 && ! exprListResult . trailingComma ; if ( ! isExpression ) { extendRange ( tupleOrExpression , startParen ) ; } if ( this . _peekTokenType ( ) !== TokenType . CloseParenthesis ) { return this . _handleExpressionParseError ( ErrorExpressionCategory . MissingTupleCloseParen , Localizer . Diagnostic . expectedCloseParen ( ) , startParen , exprListResult . parseError ?? tupleOrExpression ) ; } else { const nextToken = this . _getNextToken ( ) ; if ( ! isExpression ) { extendRange ( tupleOrExpression , nextToken ) ; } } return tupleOrExpression ; } private _parseListAtom ( ) { const startBracket = this . _getNextToken ( ) ; assert ( startBracket . type === TokenType . OpenBracket ) ; const exprListResult = this . _parseTestListWithComprehension ( ) ; const closeBracket = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfType ( TokenType . CloseBracket ) ) { return this . _handleExpressionParseError ( ErrorExpressionCategory . MissingListCloseBracket , Localizer . Diagnostic . expectedCloseBracket ( ) , startBracket , exprListResult . parseError ?? _createList ( ) ) ; } return _createList ( ) ; function _createList ( ) { const listAtom = ListNode . create ( startBracket ) ; if ( closeBracket ) { extendRange ( listAtom , closeBracket ) ; } if ( exprListResult . list . length > 0 ) { exprListResult . list . forEach ( ( expr ) => { expr . parent = listAtom ; } ) ; extendRange ( listAtom , exprListResult . list [ exprListResult . list . length - 1 ] ) ; } listAtom . entries = exprListResult . list ; return listAtom ; } } private _parseTestListWithComprehension ( ) { let sawComprehension = false ; return this . _parseExpressionListGeneric ( ( ) => { let expr = this . _parseTestOrStarExpression ( true ) ; const listComp = this . _tryParseListComprehension ( expr ) ; if ( listComp ) { expr = listComp ; sawComprehension = true ; } return expr ; } , ( ) => this . _isNextTokenNeverExpression ( ) , ( ) => sawComprehension ) ; } private _parseDictionaryOrSetAtom ( ) { const startBrace = this . _getNextToken ( ) ; assert ( startBrace . type === TokenType . OpenCurlyBrace ) ; const dictionaryEntries = [ ] ; const setEntries = [ ] ; let isDictionary = false ; let isSet = false ; let sawListComprehension = false ; let isFirstEntry = true ; let trailingCommaToken ; while ( true ) { if ( this . _peekTokenType ( ) === TokenType . CloseCurlyBrace ) { break ; } trailingCommaToken = undefined ; let doubleStarExpression ; let keyExpression ; let valueExpression ; const doubleStar = this . _peekToken ( ) ; if ( this . _consumeTokenIfOperator ( OperatorType . Power ) ) { doubleStarExpression = this . _parseExpression ( false ) ; } else { keyExpression = this . _parseTestOrStarExpression ( true ) ; if ( this . _consumeTokenIfType ( TokenType . Colon ) ) { valueExpression = this . _parseTestExpression ( false ) ; } } if ( keyExpression && valueExpression ) { if ( keyExpression . nodeType === ParseNodeType . Unpack ) { this . _addError ( Localizer . Diagnostic . unpackInDict ( ) , keyExpression ) ; } if ( isSet ) { this . _addError ( Localizer . Diagnostic . keyValueInSet ( ) , valueExpression ) ; } else { const keyEntryNode = DictionaryKeyEntryNode . create ( keyExpression , valueExpression ) ; let dictEntry = keyEntryNode ; const listComp = this . _tryParseListComprehension ( keyEntryNode ) ; if ( listComp ) { dictEntry = listComp ; sawListComprehension = true ; if ( ! isFirstEntry ) { this . _addError ( Localizer . Diagnostic . comprehensionInDict ( ) , dictEntry ) ; } } dictionaryEntries . push ( dictEntry ) ; isDictionary = true ; } } else if ( doubleStarExpression ) { if ( isSet ) { this . _addError ( Localizer . Diagnostic . unpackInSet ( ) , doubleStarExpression ) ; } else { const listEntryNode = DictionaryExpandEntryNode . create ( doubleStarExpression ) ; extendRange ( listEntryNode , doubleStar ) ; let expandEntryNode = listEntryNode ; const listComp = this . _tryParseListComprehension ( listEntryNode ) ; if ( listComp ) { expandEntryNode = listComp ; sawListComprehension = true ; if ( ! isFirstEntry ) { this . _addError ( Localizer . Diagnostic . comprehensionInDict ( ) , doubleStarExpression ) ; } } dictionaryEntries . push ( expandEntryNode ) ; isDictionary = true ; } } else { assert ( keyExpression !== undefined ) ; if ( keyExpression ) { if ( isDictionary ) { const missingValueErrorNode = ErrorNode . create ( this . _peekToken ( ) , ErrorExpressionCategory . MissingDictValue ) ; const keyEntryNode = DictionaryKeyEntryNode . create ( keyExpression , missingValueErrorNode ) ; dictionaryEntries . push ( keyEntryNode ) ; this . _addError ( Localizer . Diagnostic . dictKeyValuePairs ( ) , keyExpression ) ; } else { const listComp = this . _tryParseListComprehension ( keyExpression ) ; if ( listComp ) { keyExpression = listComp ; sawListComprehension = true ; if ( ! isFirstEntry ) { this . _addError ( Localizer . Diagnostic . comprehensionInSet ( ) , keyExpression ) ; } } setEntries . push ( keyExpression ) ; isSet = true ; } } } if ( sawListComprehension ) { break ; } if ( this . _peekTokenType ( ) !== TokenType . Comma ) { break ; } trailingCommaToken = this . _getNextToken ( ) ; isFirstEntry = false ; } let closeCurlyBrace = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfType ( TokenType . CloseCurlyBrace ) ) { this . _addError ( Localizer . Diagnostic . expectedCloseBrace ( ) , startBrace ) ; closeCurlyBrace = undefined ; } if ( isSet ) { const setAtom = SetNode . create ( startBrace ) ; if ( closeCurlyBrace ) { extendRange ( setAtom , closeCurlyBrace ) ; } if ( setEntries . length > 0 ) { extendRange ( setAtom , setEntries [ setEntries . length - 1 ] ) ; } setEntries . forEach ( ( entry ) => { entry . parent = setAtom ; } ) ; setAtom . entries = setEntries ; return setAtom ; } const dictionaryAtom = DictionaryNode . create ( startBrace ) ; if ( trailingCommaToken ) { dictionaryAtom . trailingCommaToken = trailingCommaToken ; extendRange ( dictionaryAtom , trailingCommaToken ) ; } if ( closeCurlyBrace ) { extendRange ( dictionaryAtom , closeCurlyBrace ) ; } if ( dictionaryEntries . length > 0 ) { dictionaryEntries . forEach ( ( entry ) => { entry . parent = dictionaryAtom ; } ) ; extendRange ( dictionaryAtom , dictionaryEntries [ dictionaryEntries . length - 1 ] ) ; } dictionaryAtom . entries = dictionaryEntries ; return dictionaryAtom ; } private _parseExpressionListGeneric < T extends ParseNode = ExpressionNode > ( parser , terminalCheck = ( ) => this . _isNextTokenNeverExpression ( ) , finalEntryCheck = ( ) => false ) { let trailingComma = false ; const list = [ ] ; let parseError ; while ( true ) { if ( terminalCheck ( ) ) { break ; } const expr = parser ( ) ; if ( expr . nodeType === ParseNodeType . Error ) { parseError = expr as ErrorNode ; break ; } list . push ( expr ) ; if ( finalEntryCheck ( ) ) { break ; } if ( ! this . _consumeTokenIfType ( TokenType . Comma ) ) { trailingComma = false ; break ; } trailingComma = true ; } return { trailingComma , list , parseError } ; } private _parseExpressionStatement ( ) { let leftExpr = this . _parseTestOrStarListAsExpression ( false , ErrorExpressionCategory . MissingExpression , Localizer . Diagnostic . expectedExpr ( ) ) ; let annotationExpr ; if ( leftExpr . nodeType === ParseNodeType . Error ) { return leftExpr ; } if ( this . _consumeTokenIfType ( TokenType . Colon ) ) { annotationExpr = this . _parseTypeAnnotation ( ) ; leftExpr = TypeAnnotationNode . create ( leftExpr , annotationExpr ) ; if ( ! this . _parseOptions . isStubFile && this . _getLanguageVersion ( ) < PythonVersion . V3_6 ) { this . _addError ( Localizer . Diagnostic . varAnnotationIllegal ( ) , annotationExpr ) ; } if ( ! this . _consumeTokenIfOperator ( OperatorType . Assign ) ) { return leftExpr ; } const isTypeAliasDeclaration = this . _isTypingAnnotation ( annotationExpr , 'TypeAlias' ) ; const wasParsingTypeAnnotation = this . _isParsingTypeAnnotation ; if ( isTypeAliasDeclaration ) { this . _isParsingTypeAnnotation = true ; } const rightExpr = this . _tryParseYieldExpression ( ) || this . _parseTestOrStarListAsExpression ( false , ErrorExpressionCategory . MissingExpression , Localizer . Diagnostic . expectedAssignRightHandExpr ( ) ) ; this . _isParsingTypeAnnotation = wasParsingTypeAnnotation ; return AssignmentNode . create ( leftExpr , rightExpr ) ; } if ( this . _consumeTokenIfOperator ( OperatorType . Assign ) ) { return this . _parseChainAssignments ( leftExpr ) ; } if ( Tokenizer . isOperatorAssignment ( this . _peekOperatorType ( ) ) ) { const operatorToken = this . _getNextToken ( ) as OperatorToken ; const rightExpr = this . _tryParseYieldExpression ( ) || this . _parseTestListAsExpression ( ErrorExpressionCategory . MissingExpression , Localizer . Diagnostic . expectedBinaryRightHandExpr ( ) ) ; const destExpr = Object . assign ( { } , leftExpr ) ; destExpr . id = getNextNodeId ( ) ; return AugmentedAssignmentNode . create ( leftExpr , rightExpr , operatorToken . operatorType , destExpr ) ; } return leftExpr ; } private _parseChainAssignments ( leftExpr ) { let rightExpr = this . _tryParseYieldExpression ( ) || this . _parseTestOrStarListAsExpression ( false , ErrorExpressionCategory . MissingExpression , Localizer . Diagnostic . expectedAssignRightHandExpr ( ) ) ; if ( rightExpr . nodeType === ParseNodeType . Error ) { return AssignmentNode . create ( leftExpr , rightExpr ) ; } if ( this . _consumeTokenIfOperator ( OperatorType . Assign ) ) { rightExpr = this . _parseChainAssignments ( rightExpr ) ; if ( rightExpr . nodeType === ParseNodeType . Error ) { return rightExpr ; } } const assignmentNode = AssignmentNode . create ( leftExpr , rightExpr ) ; const typeAnnotationComment = this . _parseVariableTypeAnnotationComment ( ) ; if ( typeAnnotationComment ) { assignmentNode . typeAnnotationComment = typeAnnotationComment ; assignmentNode . typeAnnotationComment . parent = assignmentNode ; extendRange ( assignmentNode , assignmentNode . typeAnnotationComment ) ; } return assignmentNode ; } private _parseFunctionTypeAnnotation ( ) { const openParenToken = this . _peekToken ( ) ; if ( ! this . _consumeTokenIfType ( TokenType . OpenParenthesis ) ) { this . _addError ( Localizer . Diagnostic . expectedOpenParen ( ) , this . _peekToken ( ) ) ; return undefined ; } let paramAnnotations = [ ] ; while ( true ) { const nextTokenType = this . _peekTokenType ( ) ; if ( nextTokenType === TokenType . CloseParenthesis || nextTokenType === TokenType . NewLine || nextTokenType === TokenType . EndOfStream ) { break ; } this . _consumeTokenIfOperator ( OperatorType . Multiply ) || this . _consumeTokenIfOperator ( OperatorType . Power ) ; const paramAnnotation = this . _parseTypeAnnotation ( ) ; paramAnnotations . push ( paramAnnotation ) ; if ( ! this . _consumeTokenIfType ( TokenType . Comma ) ) { break ; } } if ( ! this . _consumeTokenIfType ( TokenType . CloseParenthesis ) ) { this . _addError ( Localizer . Diagnostic . expectedCloseParen ( ) , openParenToken ) ; this . _consumeTokensUntilType ( [ TokenType . Colon ] ) ; } if ( ! this . _consumeTokenIfType ( TokenType . Arrow ) ) { this . _addError ( Localizer . Diagnostic . expectedArrow ( ) , this . _peekToken ( ) ) ; return undefined ; } const returnType = this . _parseTypeAnnotation ( ) ; let isParamListEllipsis = false ; if ( paramAnnotations . length === 1 && paramAnnotations [ 0 ] . nodeType === ParseNodeType . Ellipsis ) { paramAnnotations = [ ] ; isParamListEllipsis = true ; } return FunctionAnnotationNode . create ( openParenToken , isParamListEllipsis , paramAnnotations , returnType ) ; } private _parseTypeAnnotation ( ) { const wasParsingTypeAnnotation = this . _isParsingTypeAnnotation ; this . _isParsingTypeAnnotation = true ; const startToken = this . _peekToken ( ) ; const isUnpack = this . _consumeTokenIfOperator ( OperatorType . Multiply ) ; let result = this . _parseTestExpression ( false ) ; if ( isUnpack ) { result = UnpackNode . create ( startToken , result ) ; } this . _isParsingTypeAnnotation = wasParsingTypeAnnotation ; return result ; } private _reportStringTokenErrors ( stringToken , unescapedResult ) { if ( stringToken . flags & StringTokenFlags . Unterminated ) { this . _addError ( Localizer . Diagnostic . stringUnterminated ( ) , stringToken ) ; } if ( unescapedResult . nonAsciiInBytes ) { this . _addError ( Localizer . Diagnostic . stringNonAsciiBytes ( ) , stringToken ) ; } if ( stringToken . flags & StringTokenFlags . Format ) { if ( this . _getLanguageVersion ( ) < PythonVersion . V3_6 ) { this . _addError ( Localizer . Diagnostic . formatStringIllegal ( ) , stringToken ) ; } if ( stringToken . flags & StringTokenFlags . Bytes ) { this . _addError ( Localizer . Diagnostic . formatStringBytes ( ) , stringToken ) ; } if ( stringToken . flags & StringTokenFlags . Unicode ) { this . _addError ( Localizer . Diagnostic . formatStringUnicode ( ) , stringToken ) ; } } } private _makeStringNode ( stringToken ) { const unescapedResult = StringTokenUtils . getUnescapedString ( stringToken ) ; this . _reportStringTokenErrors ( stringToken , unescapedResult ) ; return StringNode . create ( stringToken , unescapedResult . value , unescapedResult . unescapeErrors . length > 0 ) ; } private _getTypeAnnotationCommentText ( ) { if ( this . _tokenIndex === 0 ) { return undefined ; } const curToken = this . _tokenizerOutput ! . tokens . getItemAt ( this . _tokenIndex - 1 ) ; const nextToken = this . _tokenizerOutput ! . tokens . getItemAt ( this . _tokenIndex ) ; if ( curToken . start + curToken . length === nextToken . start ) { return undefined ; } const interTokenContents = this . _fileContents ! . substring ( curToken . start + curToken . length , nextToken . start ) ; const commentRegEx = /^(\s*#\s*type:\s*)([^\r\n]*)/ ; const match = interTokenContents . match ( commentRegEx ) ; if ( ! match ) { return undefined ; } const typeString = match [ 2 ] ; if ( typeString . trim ( ) . match ( /^ignore(\s|\[|$)/ ) ) { return undefined ; } const tokenOffset = curToken . start + curToken . length + match [ 1 ] . length ; return StringToken . create ( tokenOffset , typeString . length , StringTokenFlags . None , typeString , 0 , undefined ) ; } private _parseVariableTypeAnnotationComment ( ) { const stringToken = this . _getTypeAnnotationCommentText ( ) ; if ( ! stringToken ) { return undefined ; } const stringNode = this . _makeStringNode ( stringToken ) ; const stringListNode = StringListNode . create ( [ stringNode ] ) ; const parser = new Parser ( ) ; const parseResults = parser . parseTextExpression ( this . _fileContents ! , stringToken . start , stringToken . length , this . _parseOptions , ParseTextMode . VariableAnnotation ) ; parseResults . diagnostics . forEach ( ( diag ) => { this . _addError ( diag . message , stringListNode ) ; } ) ; if ( ! parseResults . parseTree ) { return undefined ; } return parseResults . parseTree ; } private _parseFunctionTypeAnnotationComment ( stringToken , functionNode ) { const stringNode = this . _makeStringNode ( stringToken ) ; const stringListNode = StringListNode . create ( [ stringNode ] ) ; const parser = new Parser ( ) ; const parseResults = parser . parseTextExpression ( this . _fileContents ! , stringToken . start , stringToken . length , this . _parseOptions , ParseTextMode . FunctionAnnotation ) ; parseResults . diagnostics . forEach ( ( diag ) => { this . _addError ( diag . message , stringListNode ) ; } ) ; if ( ! parseResults . parseTree || parseResults . parseTree . nodeType !== ParseNodeType . FunctionAnnotation ) { return ; } const functionAnnotation = parseResults . parseTree ; functionNode . functionAnnotationComment = functionAnnotation ; functionAnnotation . parent = functionNode ; extendRange ( functionNode , functionAnnotation ) ; } private _parseFormatStringSegment ( stringToken , segment , segmentOffset , segmentLength ) { assert ( segment . isExpression ) ; const parser = new Parser ( ) ; const parseResults = parser . parseTextExpression ( this . _fileContents ! , stringToken . start + stringToken . prefixLength + stringToken . quoteMarkLength + segment . offset + segmentOffset , segmentLength , this . _parseOptions , ParseTextMode . Expression , 1 ) ; parseResults . diagnostics . forEach ( ( diag ) => { const textRangeStart = ( diag . range ? convertPositionToOffset ( diag . range . start , parseResults . lines ) : stringToken . start ) || stringToken . start ; const textRangeEnd = ( diag . range ? ( convertPositionToOffset ( diag . range . end , parseResults . lines ) || 0 ) + 1 : stringToken . start + stringToken . length ) || stringToken . start + stringToken . length ; const textRange = { start : textRangeStart , length : textRangeEnd - textRangeStart } ; this . _addError ( diag . message , textRange ) ; } ) ; return parseResults . parseTree ; } private _parseFormatString ( stringToken ) { const unescapedResult = StringTokenUtils . getUnescapedString ( stringToken ) ; this . _reportStringTokenErrors ( stringToken , unescapedResult ) ; const formatExpressions = [ ] ; for ( const segment of unescapedResult . formatStringSegments ) { if ( segment . isExpression ) { const segmentExprLength = this . _getFormatStringExpressionLength ( segment . value . trimEnd ( ) ) ; const parseTree = this . _parseFormatStringSegment ( stringToken , segment , 0 , segmentExprLength ) ; if ( parseTree ) { formatExpressions . push ( parseTree ) ; } const formatDirective = segment . value . substr ( segmentExprLength ) ; let braceDepth = 0 ; let startOfExprOffset = 0 ; for ( let i = 0 ; i < formatDirective . length ; i ++ ) { if ( formatDirective . charCodeAt ( i ) === Char . OpenBrace ) { if ( braceDepth === 0 ) { startOfExprOffset = i + 1 ; } braceDepth ++ ; } else if ( formatDirective . charCodeAt ( i ) === Char . CloseBrace ) { if ( braceDepth > 0 ) { braceDepth -- ; if ( braceDepth === 0 ) { const formatSegmentLength = this . _getFormatStringExpressionLength ( segment . value . substr ( segmentExprLength + startOfExprOffset , i - startOfExprOffset ) ) ; const parseTree = this . _parseFormatStringSegment ( stringToken , segment , segmentExprLength + startOfExprOffset , formatSegmentLength ) ; if ( parseTree ) { formatExpressions . push ( parseTree ) ; } } } } } } } return FormatStringNode . create ( stringToken , unescapedResult . value , unescapedResult . unescapeErrors . length > 0 , formatExpressions ) ; } private _getFormatStringExpressionLength ( segmentValue ) { let segmentExprLength = 0 ; const quoteStack = [ ] ; let braceCount = 0 ; let parenCount = 0 ; let bracketCount = 0 ; let indexOfDebugEqual ; while ( segmentExprLength < segmentValue . length ) { const curChar = segmentValue [ segmentExprLength ] ; const ignoreSeparator = quoteStack . length > 0 || braceCount > 0 || parenCount > 0 || bracketCount > 0 ; const inString = quoteStack . length > 0 ; if ( curChar === '=' ) { indexOfDebugEqual = segmentExprLength ; } else { if ( curChar === ':' ) { if ( ! ignoreSeparator ) { break ; } } else if ( curChar === '!' ) { if ( ! ignoreSeparator ) { if ( segmentExprLength === segmentValue . length - 1 || segmentValue [ segmentExprLength + 1 ] !== '=' ) { break ; } } } else if ( curChar === "'" || curChar === '"' ) { let quoteSequence = curChar ; if ( segmentExprLength + 2 < segmentValue . length && segmentValue [ segmentExprLength + 1 ] === curChar && segmentValue [ segmentExprLength + 2 ] === curChar ) { quoteSequence = curChar + curChar + curChar ; segmentExprLength += 2 ; } if ( quoteStack . length > 0 && quoteStack [ quoteStack . length - 1 ] === quoteSequence ) { quoteStack . pop ( ) ; } else if ( quoteStack . length === 0 ) { quoteStack . push ( quoteSequence ) ; } } else if ( curChar === '(' ) { if ( ! inString ) { parenCount ++ ; } } else if ( curChar === ')' ) { if ( ! inString && parenCount > 0 ) { parenCount -- ; } } else if ( curChar === '{' ) { if ( ! inString ) { braceCount ++ ; } } else if ( curChar === '}' ) { if ( ! inString && braceCount > 0 ) { braceCount -- ; } } else if ( curChar === '[' ) { if ( ! inString ) { bracketCount ++ ; } } else if ( curChar === ']' ) { if ( ! inString && bracketCount > 0 ) { bracketCount -- ; } } if ( curChar !== ' ' ) { indexOfDebugEqual = undefined ; } } segmentExprLength ++ ; } if ( this . _parseOptions . pythonVersion >= PythonVersion . V3_8 && indexOfDebugEqual !== undefined ) { segmentExprLength = indexOfDebugEqual ; } return segmentExprLength ; } private _parseStringList ( ) { const stringList = [ ] ; while ( this . _peekTokenType ( ) === TokenType . String ) { const stringToken = this . _getNextToken ( ) as StringToken ; if ( stringToken . flags & StringTokenFlags . Format ) { stringList . push ( this . _parseFormatString ( stringToken ) ) ; } else { stringList . push ( this . _makeStringNode ( stringToken ) ) ; } } const stringNode = StringListNode . create ( stringList ) ; if ( this . _isParsingTypeAnnotation ) { if ( stringNode . strings . length > 1 ) { this . _addError ( Localizer . Diagnostic . annotationSpansStrings ( ) , stringNode ) ; } else if ( stringNode . strings [ 0 ] . token . flags & StringTokenFlags . Format ) { this . _addError ( Localizer . Diagnostic . annotationFormatString ( ) , stringNode ) ; } else { const stringToken = stringNode . strings [ 0 ] . token ; const stringValue = StringTokenUtils . getUnescapedString ( stringNode . strings [ 0 ] . token ) ; const unescapedString = stringValue . value ; const tokenOffset = stringToken . start ; const prefixLength = stringToken . prefixLength + stringToken . quoteMarkLength ; if ( unescapedString . length !== stringToken . length - prefixLength - stringToken . quoteMarkLength ) { this . _addError ( Localizer . Diagnostic . annotationStringEscape ( ) , stringNode ) ; } else { const parser = new Parser ( ) ; const parseResults = parser . parseTextExpression ( this . _fileContents ! , tokenOffset + prefixLength , unescapedString . length , this . _parseOptions , ParseTextMode . VariableAnnotation , ( stringNode . strings [ 0 ] . token . flags & StringTokenFlags . Triplicate ) !== 0 ? 1 : 0 ) ; parseResults . diagnostics . forEach ( ( diag ) => { this . _addError ( diag . message , stringNode ) ; } ) ; if ( parseResults . parseTree ) { stringNode . typeAnnotation = parseResults . parseTree ; stringNode . typeAnnotation . parent = stringNode ; } } } } return stringNode ; } private _reportConditionalErrorForStarTupleElement ( possibleTupleExpr ) { if ( possibleTupleExpr . nodeType !== ParseNodeType . Tuple ) { return ; } if ( possibleTupleExpr . enclosedInParens ) { return ; } if ( this . _parseOptions . pythonVersion >= PythonVersion . V3_8 ) { return ; } for ( const expr of possibleTupleExpr . expressions ) { if ( expr . nodeType === ParseNodeType . Unpack ) { this . _addError ( Localizer . Diagnostic . unpackTuplesIllegal ( ) , expr ) ; return ; } } } private _isNextTokenNeverExpression ( ) { const nextToken = this . _peekToken ( ) ; switch ( nextToken . type ) { case TokenType . Keyword : { switch ( this . _peekKeywordType ( ) ) { case KeywordType . For : case KeywordType . In : case KeywordType . If : return true ; } break ; } case TokenType . Operator : { switch ( this . _peekOperatorType ( ) ) { case OperatorType . AddEqual : case OperatorType . SubtractEqual : case OperatorType . MultiplyEqual : case OperatorType . DivideEqual : case OperatorType . ModEqual : case OperatorType . BitwiseAndEqual : case OperatorType . BitwiseOrEqual : case OperatorType . BitwiseXorEqual : case OperatorType . LeftShiftEqual : case OperatorType . RightShiftEqual : case OperatorType . PowerEqual : case OperatorType . FloorDivideEqual : case OperatorType . Assign : return true ; } break ; } case TokenType . Indent : case TokenType . Dedent : case TokenType . NewLine : case TokenType . EndOfStream : case TokenType . Semicolon : case TokenType . CloseParenthesis : case TokenType . CloseBracket : case TokenType . CloseCurlyBrace : case TokenType . Comma : case TokenType . Colon : return true ; } return false ; } private _disallowAssignmentExpression ( callback ) { const wasAllowed = this . _assignmentExpressionsAllowed ; this . _assignmentExpressionsAllowed = false ; callback ( ) ; this . _assignmentExpressionsAllowed = wasAllowed ; } private _getNextToken ( ) { const token = this . _tokenizerOutput ! . tokens . getItemAt ( this . _tokenIndex ) ; if ( ! this . _atEof ( ) ) { this . _tokenIndex ++ ; } return token ; } private _atEof ( ) { return this . _tokenIndex >= this . _tokenizerOutput ! . tokens . count - 1 ; } private _peekToken ( count = 0 ) { if ( this . _tokenIndex + count < 0 ) { return this . _tokenizerOutput ! . tokens . getItemAt ( 0 ) ; } if ( this . _tokenIndex + count >= this . _tokenizerOutput ! . tokens . count ) { return this . _tokenizerOutput ! . tokens . getItemAt ( this . _tokenizerOutput ! . tokens . count - 1 ) ; } return this . _tokenizerOutput ! . tokens . getItemAt ( this . _tokenIndex + count ) ; } private _peekTokenType ( ) { return this . _peekToken ( ) . type ; } private _peekKeywordType ( ) { const nextToken = this . _peekToken ( ) ; if ( nextToken . type !== TokenType . Keyword ) { return undefined ; } return ( nextToken as KeywordToken ) . keywordType ; } private _peekOperatorType ( ) { const nextToken = this . _peekToken ( ) ; if ( nextToken . type !== TokenType . Operator ) { return undefined ; } return ( nextToken as OperatorToken ) . operatorType ; } private _getTokenIfIdentifier ( ) { const nextToken = this . _peekToken ( ) ; if ( nextToken . type === TokenType . Identifier ) { return this . _getNextToken ( ) as IdentifierToken ; } if ( nextToken . type === TokenType . Invalid ) { this . _getNextToken ( ) ; this . _addError ( Localizer . Diagnostic . invalidIdentifierChar ( ) , nextToken ) ; return IdentifierToken . create ( nextToken . start , nextToken . length , '' , nextToken . comments ) ; } if ( nextToken . type === TokenType . Keyword ) { const keywordType = this . _peekKeywordType ( ) ; const softKeywords = [ KeywordType . Debug , KeywordType . Match , KeywordType . Case ] ; if ( softKeywords . find ( ( type ) => type === keywordType ) ) { const keywordText = this . _fileContents ! . substr ( nextToken . start , nextToken . length ) ; this . _getNextToken ( ) ; return IdentifierToken . create ( nextToken . start , nextToken . length , keywordText , nextToken . comments ) ; } } return undefined ; } private _consumeTokensUntilType ( terminators ) { while ( true ) { const token = this . _peekToken ( ) ; if ( terminators . some ( ( term ) => term === token . type ) ) { return true ; } if ( token . type === TokenType . EndOfStream ) { return false ; } this . _getNextToken ( ) ; } } private _getTokenIfType ( tokenType ) { if ( this . _peekTokenType ( ) === tokenType ) { return this . _getNextToken ( ) ; } return undefined ; } private _consumeTokenIfType ( tokenType ) { return ! ! this . _getTokenIfType ( tokenType ) ; } private _consumeTokenIfKeyword ( keywordType ) { if ( this . _peekKeywordType ( ) === keywordType ) { this . _getNextToken ( ) ; return true ; } return false ; } private _consumeTokenIfOperator ( operatorType ) { if ( this . _peekOperatorType ( ) === operatorType ) { this . _getNextToken ( ) ; return true ; } return false ; } private _getKeywordToken ( keywordType ) { const keywordToken = this . _getNextToken ( ) as KeywordToken ; assert ( keywordToken . type === TokenType . Keyword ) ; assert ( keywordToken . keywordType === keywordType ) ; return keywordToken ; } private _getLanguageVersion ( ) { return this . _parseOptions . pythonVersion ; } private _suppressErrors ( callback ) { const errorsWereSuppressed = this . _areErrorsSuppressed ; try { this . _areErrorsSuppressed = true ; callback ( ) ; } finally { this . _areErrorsSuppressed = errorsWereSuppressed ; } } private _addError ( message , range ) { assert ( range !== undefined ) ; if ( ! this . _areErrorsSuppressed ) { this . _diagSink . addError ( message , convertOffsetsToRange ( range . start , range . start + range . length , this . _tokenizerOutput ! . lines ) ) ; } } }