import Char from 'typescript-char' ; import { findNodeByOffset , getFirstAncestorOrSelf , getFirstAncestorOrSelfOfKind , getStringValueRange , getTokenAt , isDocString , } from '../analyzer/parseTreeUtils' ; import { convertOffsetToPosition , convertTextRangeToRange } from '../common/positionUtils' ; import { Range , TextRange } from '../common/textRange' ; import { TextRangeCollection } from '../common/textRangeCollection' ; import { ModuleNode , ParseNode , ParseNodeType , SuiteNode } from '../parser/parseNodes' ; import { ParseResults } from '../parser/parser' ; import { defaultTabSize } from '../parser/tokenizer' ; import { IndentToken , KeywordToken , KeywordType , StringToken , StringTokenFlags , Token , TokenType , } from '../parser/tokenizerTypes' ; interface TokenInfo extends TextRange { range : Range ; text : string ; kind : 'comment' | 'string' | 'token' ; firstTokenOnLine : boolean ; multilineDocComment : boolean ; } export function getIndentation ( parseResults , offset , preferDedent ? ) { const strIndent = _tryHandleStringLiterals ( parseResults , offset ) ; if ( strIndent !== undefined ) { return strIndent ; } const exprIndent = _tryHandleMultilineConstructs ( parseResults , offset ) ; if ( exprIndent !== undefined ) { return exprIndent ; } preferDedent = preferDedent ?? _shouldDedentAfterKeyword ( parseResults . tokenizerOutput . tokens , offset ) ; return Math . max ( _getIndentation ( parseResults , offset , preferDedent ) . indentation , 0 ) ; } export function reindentSpan ( parseResults , span , indentation , indentFirstToken = true ) { let indentDelta = 0 ; const texts = [ ] ; const tokenInfo = _convertTokenStreams ( parseResults , span ) ; let previousInfo = tokenInfo [ 0 ] ; indentDelta = indentation - _getIndentationFromText ( parseResults , previousInfo . range . start . line , previousInfo . range . start . character ) . indentation ; if ( previousInfo . multilineDocComment ) { texts . push ( ... _reindentLinesFromText ( parseResults , previousInfo , indentDelta ) ) ; } else { if ( indentFirstToken ) { texts . push ( _createIndentationString ( parseResults , indentation ) ) ; } texts . push ( previousInfo . text ) ; } for ( let i = 1 ; i < tokenInfo . length ; i ++ ) { const info = tokenInfo [ i ] ; if ( info . firstTokenOnLine ) { texts . push ( parseResults . tokenizerOutput . predominantEndOfLineSequence . repeat ( info . range . start . line - previousInfo . range . end . line ) ) ; if ( info . multilineDocComment ) { texts . push ( ... _reindentLinesFromText ( parseResults , info , indentDelta ) ) ; } else { texts . push ( _createIndentationString ( parseResults , Math . max ( 0 , _getIndentationFromText ( parseResults , info . range . start . line , info . range . start . character ) . indentation + indentDelta ) ) ) ; texts . push ( info . text ) ; } } else { texts . push ( ' ' . repeat ( info . range . start . character - previousInfo . range . end . character ) ) ; texts . push ( info . text ) ; } previousInfo = info ; } return texts . join ( '' ) ; } function _getIndentation ( parseResults , offset , preferDedent ) { const tokens = parseResults . tokenizerOutput . tokens ; const startingToken = _findPreviousNonWhitespaceToken ( tokens , offset ) ; if ( ! startingToken ) { return { indentation : 0 , } ; } const node = findNodeByOffset ( parseResults . parseTree , TextRange . getEnd ( startingToken ) ) ; if ( ! node ) { return { indentation : 0 , } ; } const suite = getFirstAncestorOrSelfOfKind ( node , ParseNodeType . Suite ) ; if ( ! suite ) { return _getIndentationForNode ( parseResults , parseResults . parseTree , node ) ; } const suiteSpan = convertTextRangeToRange ( suite , parseResults . tokenizerOutput . lines ) ; if ( preferDedent || suiteSpan . start . line === suiteSpan . end . line ) { const outerContainer = getFirstAncestorOrSelf ( suite , ( n ) => n !== suite && n . nodeType === ParseNodeType . Suite ) as SuiteNode | undefined ; return _getIndentationForNode ( parseResults , outerContainer ?? parseResults . parseTree , suite ) ; } return _getIndentationForNode ( parseResults , suite , node ) ; } function _getIndentationForNode ( parseResults , container , current ) { if ( container . nodeType === ParseNodeType . Module ) { return { token : _getFirstTokenOFStatement ( parseResults , container , current ) , indentation : 0 , } ; } if ( _containsNoIndentBeforeFirstStatement ( parseResults , container ) ) { const tabSize = _getTabSize ( parseResults ) ; const outerContainer = getFirstAncestorOrSelf ( container , ( n ) => n !== container && n . nodeType === ParseNodeType . Suite ) as SuiteNode | undefined ; const result = _getIndentationForNode ( parseResults , outerContainer ?? parseResults . parseTree , container ) ; return { token : result . token , indentation : result . indentation + tabSize , } ; } else { const tokens = parseResults . tokenizerOutput . tokens ; return { token : _getFirstTokenOFStatement ( parseResults , container , current ) , indentation : _getIndentationFromIndentToken ( tokens , tokens . getItemAtPosition ( container . start ) ) , } ; } } function _containsNoIndentBeforeFirstStatement ( parseResults , suite ) { if ( suite . statements . filter ( ( s ) => s . length > 0 ) . length === 0 ) { return true ; } const tokens = parseResults . tokenizerOutput . tokens ; const start = tokens . getItemAtPosition ( suite . start ) ; const end = tokens . getItemAtPosition ( suite . statements [ 0 ] . start ) ; for ( let i = start ; i <= end ; i ++ ) { const token = _getTokenAtIndex ( tokens , i ) ; if ( token ?. type === TokenType . Indent ) { return false ; } } return true ; } function _getFirstTokenOFStatement ( parseResults , container , span ) { const tokens = parseResults . tokenizerOutput . tokens ; for ( const statement of container . statements ) { if ( ! TextRange . containsRange ( statement , span ) ) { continue ; } return getTokenAt ( tokens , statement . start ) ! ; } return getTokenAt ( tokens , container . start ) ! ; } function _getIndentationFromIndentToken ( tokens , index ) { for ( let i = index ; i < tokens . count ; i ++ ) { const token = _getTokenAtIndex ( tokens , i ) ; if ( token ?. type === TokenType . Indent ) { return ( token as IndentToken ) . indentAmount ; } } return 0 ; } function _tryHandleMultilineConstructs ( parseResults , offset ) { const tokens = parseResults . tokenizerOutput . tokens ; const index = _findNextTokenIndex ( tokens , offset ) ; if ( index < 0 ) { return undefined ; } const lines = parseResults . tokenizerOutput . lines ; const tabSize = _getTabSize ( parseResults ) ; for ( let i = index ; i > 0 ; i -- ) { const token = _getTokenAtIndex ( tokens , i ) ! ; if ( TextRange . getEnd ( token ) < offset ) { return undefined ; } const previousToken = _getTokenAtIndex ( tokens , i - 1 ) ! ; const tokenSpan = token ? convertTextRangeToRange ( token , lines ) : undefined ; const previousTokenSpan = previousToken ? convertTextRangeToRange ( previousToken , lines ) : undefined ; if ( tokenSpan && previousTokenSpan && previousTokenSpan . end . line < tokenSpan . start . line && previousToken ! . type !== TokenType . NewLine ) { const indentationResult = _getIndentation ( parseResults , previousToken ! . start , false ) ; const currentPosition = convertOffsetToPosition ( offset , lines ) ; const lineDelta = currentPosition . line - ( indentationResult . token ? convertOffsetToPosition ( indentationResult . token . start , lines ) . line : previousTokenSpan . start . line ) ; const indentation = _getFirstNonBlankLineIndentationFromText ( parseResults , currentPosition . line , previousTokenSpan . start . line ) ; return indentation + ( lineDelta === 1 ? tabSize : 0 ) ; } } return undefined ; } function _tryHandleStringLiterals ( parseResults , offset ) { const tokens = parseResults . tokenizerOutput . tokens ; const index = tokens . getItemAtPosition ( offset ) ; if ( index < 0 ) { return undefined ; } const token = _findStringToken ( tokens , index ) ; if ( ! token || token . type !== TokenType . String ) { return undefined ; } const stringToken = token as StringToken ; if ( ! ( stringToken . flags & StringTokenFlags . Triplicate ) ) { return undefined ; } if ( ! ( stringToken . flags & StringTokenFlags . Unterminated ) && ! TextRange . contains ( getStringValueRange ( stringToken ) , offset ) ) { return undefined ; } const lines = parseResults . tokenizerOutput . lines ; const begin = convertOffsetToPosition ( token . start , lines ) ; const current = convertOffsetToPosition ( offset , lines ) ; return _getFirstNonBlankLineIndentationFromText ( parseResults , current . line , begin . line ) ; } function _getFirstNonBlankLineIndentationFromText ( parseResults , currentLine , endingLine ) { endingLine = Math . max ( endingLine , 0 ) ; for ( let i = currentLine ; i >= endingLine ; i -- ) { const result = _getIndentationFromText ( parseResults , i ) ; if ( ! _isBlankLine ( parseResults , i , result . charOffset ) ) { return result . indentation ; } } return _getIndentationFromText ( parseResults , endingLine ) . indentation ; } function _findStringToken ( tokens , index ) { const token = _findPreviousNonWhitespaceTokenFromIndex ( tokens , index ) ; if ( ! token ) { return undefined ; } return token . type === TokenType . String ? token : undefined ; } function _findPreviousNonWhitespaceToken ( tokens , offset ) { const index = tokens . getItemAtPosition ( offset ) ; if ( index < 0 ) { return undefined ; } return _findPreviousNonWhitespaceTokenFromIndex ( tokens , index ) ; } function _findPreviousNonWhitespaceTokenFromIndex ( tokens , index ) { for ( let i = index ; i >= 0 ; i -- ) { const token = _getTokenAtIndex ( tokens , i ) ; if ( ! token ) { break ; } if ( _isWhitespaceToken ( token . type ) ) { continue ; } return token ; } return undefined ; } function _findNextTokenIndex ( tokens , offset ) { const index = tokens . getItemAtPosition ( offset ) ; if ( index < 0 ) { return index ; } for ( let i = index + 1 ; i < tokens . count ; i ++ ) { const token = _getTokenAtIndex ( tokens , i ) ; if ( token ?. type === TokenType . Dedent || token ?. type === TokenType . NewLine ) { continue ; } return i ; } return tokens . count - 1 ; } function _getTokenAtIndex ( tokens , index ) { if ( index < 0 ) { return undefined ; } return tokens . getItemAt ( index ) ; } function _shouldDedentAfterKeyword ( tokens , offset ) { const index = tokens . getItemAtPosition ( offset ) ; if ( index < 0 ) { return false ; } for ( let i = index ; i >= 0 ; i -- ) { const token = _getTokenAtIndex ( tokens , i ) ; if ( ! token ) { return false ; } switch ( token . type ) { case TokenType . Dedent : case TokenType . NewLine : case TokenType . EndOfStream : continue ; case TokenType . Keyword : { const previousToken = _getTokenAtIndex ( tokens , i - 1 ) ; if ( previousToken ?. type === TokenType . Colon ) { return false ; } const keyword = token as KeywordToken ; return ( keyword . keywordType === KeywordType . Pass || keyword . keywordType === KeywordType . Return || keyword . keywordType === KeywordType . Break || keyword . keywordType === KeywordType . Continue || keyword . keywordType === KeywordType . Raise ) ; } default : return false ; } } return false ; } function _isBlankLine ( parseResults , line , charOffset ) { const endingLength = _getLineEndingLength ( parseResults , line ) ; const lineSpan = parseResults . tokenizerOutput . lines . getItemAt ( line ) ; return charOffset === lineSpan . length - endingLength ; } function _getLineEndingLength ( parseResults , line ) { let length = 0 ; const range = parseResults . tokenizerOutput . lines . getItemAt ( line ) ; for ( let i = range . length - 1 ; i >= 0 ; i -- ) { const charCode = parseResults . text . charCodeAt ( range . start + i ) ; switch ( charCode ) { case Char . FormFeed : case Char . Hash : case Char . LineFeed : case Char . CarriageReturn : length ++ ; break ; default : return length ; } } return length ; } function _getIndentationFromText ( parseResults , line , uptoLineOffset ? ) { let indentation = 0 ; let charOffset = 0 ; const tabSize = _getTabSize ( parseResults ) ; const range = parseResults . tokenizerOutput . lines . getItemAt ( line ) ; for ( let i = 0 ; i < range . length ; i ++ ) { const charCode = parseResults . text . charCodeAt ( range . start + i ) ; switch ( charCode ) { case Char . Space : charOffset ++ ; indentation ++ ; break ; case Char . Tab : charOffset ++ ; indentation += tabSize ; break ; default : if ( ! uptoLineOffset || uptoLineOffset === i ) { return { charOffset , indentation , } ; } charOffset ++ ; indentation ++ ; } } return { charOffset , indentation , } ; } function _convertTokenStreams ( parseResults , span ) { const tokens = parseResults . tokenizerOutput . tokens ; let startIndex = Math . max ( tokens . getItemAtPosition ( span . start ) , 0 ) ; const startToken = _getTokenAtIndex ( tokens , startIndex ) ! ; if ( TextRange . getEnd ( startToken ) < span . start ) { startIndex ++ ; } let endIndex = Math . min ( tokens . getItemAtPosition ( TextRange . getEnd ( span ) ) , tokens . length - 1 ) ; const endToken = _getTokenAtIndex ( tokens , endIndex ) ! ; if ( TextRange . getEnd ( span ) < endToken . start ) { endIndex -- ; } const tokenInfoArray = [ ] ; const lines = parseResults . tokenizerOutput . lines ; for ( let i = startIndex ; i <= endIndex ; i ++ ) { const token = _getTokenAtIndex ( tokens , i ) ! ; if ( token . comments ) { for ( const comment of token . comments ) { tokenInfoArray . push ( { start : comment . start , length : comment . length , range : convertTextRangeToRange ( comment , lines ) , text : comment . value , kind : 'comment' , firstTokenOnLine : false , multilineDocComment : false , } ) ; } } if ( _isWhitespaceToken ( token . type ) || token . length === 0 ) { continue ; } tokenInfoArray . push ( { start : token . start , length : token . length , range : convertTextRangeToRange ( token , lines ) , text : parseResults . text . substr ( token . start , token . length ) , kind : token . type === TokenType . String ? 'string' : 'token' , firstTokenOnLine : false , multilineDocComment : false , } ) ; } if ( tokenInfoArray . length === 0 ) { return tokenInfoArray ; } tokenInfoArray . sort ( ( a , b ) => a . start - b . start ) ; let previousInfo = tokenInfoArray [ 0 ] ; const additionalTokens = [ ] ; for ( let i = 1 ; i < tokenInfoArray . length ; i ++ ) { const info = tokenInfoArray [ i ] ; if ( previousInfo . kind !== 'comment' ) { for ( let whitespaceLine = previousInfo . range . end . line ; whitespaceLine < info . range . start . line ; whitespaceLine ++ ) { const lineTextRange = lines . getItemAt ( whitespaceLine ) ; const lastCharOffset = lineTextRange . length - _getLineEndingLength ( parseResults , whitespaceLine ) - 1 ; if ( lastCharOffset >= 0 ) { const start = lineTextRange . start + lastCharOffset ; _addTokenInfoIfMatch ( parseResults , start , start + 1 , Char . Backslash , additionalTokens ) ; } } } if ( info . kind === 'comment' ) { const start = previousInfo . range . end . line === info . range . start . line ? TextRange . getEnd ( previousInfo ) : lines . getItemAt ( info . range . start . line ) . start ; _addTokenInfoIfMatch ( parseResults , start , info . start , Char . Hash , additionalTokens ) ; } previousInfo = info ; } tokenInfoArray . push ( ... additionalTokens ) ; tokenInfoArray . sort ( ( a , b ) => a . start - b . start ) ; previousInfo = tokenInfoArray [ 0 ] ; if ( startIndex === 0 ) { previousInfo . firstTokenOnLine = true ; } else { const previousToken = _findPreviousNonWhitespaceTokenFromIndex ( tokens , startIndex - 1 ) ! ; const previousEnd = convertOffsetToPosition ( TextRange . getEnd ( previousToken ) , lines ) ; previousInfo . firstTokenOnLine = previousEnd . line !== previousInfo . range . start . line ; } previousInfo . multilineDocComment = _isMultilineDocComment ( parseResults , previousInfo ) ; for ( let i = 1 ; i < tokenInfoArray . length ; i ++ ) { const info = tokenInfoArray [ i ] ; info . firstTokenOnLine = previousInfo . range . end . line !== info . range . start . line ; info . multilineDocComment = _isMultilineDocComment ( parseResults , info ) ; previousInfo = info ; } return tokenInfoArray ; } function _addTokenInfoIfMatch ( parseResults , start , end , charCode , tokens ) { for ( let i = start ; i < end ; i ++ ) { if ( parseResults . text . charCodeAt ( i ) === charCode ) { tokens . push ( { start : i , length : 1 , range : convertTextRangeToRange ( TextRange . create ( i , 1 ) , parseResults . tokenizerOutput . lines ) , text : String . fromCharCode ( charCode ) , kind : 'token' , firstTokenOnLine : false , multilineDocComment : false , } ) ; } } } function _isWhitespaceToken ( type ) { switch ( type ) { case TokenType . Dedent : case TokenType . NewLine : case TokenType . Indent : case TokenType . EndOfStream : return true ; default : return false ; } } function _isMultilineDocComment ( parseResults , info ) { if ( info . kind !== 'string' || ! info . firstTokenOnLine || info . range . start . line === info . range . end . line ) { return false ; } const node = findNodeByOffset ( parseResults . parseTree , info . start ) ; if ( node ?. nodeType !== ParseNodeType . String || node . parent ?. nodeType !== ParseNodeType . StringList || node . parent . parent ?. nodeType !== ParseNodeType . StatementList ) { return false ; } return isDocString ( node . parent . parent ) ; } function _reindentLinesFromText ( parseResults , info , indentDelta ) { const texts = [ ] ; for ( let i = info . range . start . line ; i <= info . range . end . line ; i ++ ) { texts . push ( _reindentLineFromText ( parseResults , i , indentDelta , i === info . range . end . line ? info : undefined ) ) ; } return texts ; } function _reindentLineFromText ( parseResults , line , indentDelta , range ? ) { const result = _getIndentationFromText ( parseResults , line ) ; if ( _isBlankLine ( parseResults , line , result . charOffset ) ) { return '' ; } let lineRange = parseResults . tokenizerOutput . lines . getItemAt ( line ) ; if ( range ) { lineRange = TextRange . fromBounds ( lineRange . start , Math . min ( TextRange . getEnd ( range ) , TextRange . getEnd ( lineRange ) ) ) ; } const text = parseResults . text . substr ( lineRange . start + result . charOffset , lineRange . length - result . charOffset ) ; return _createIndentationString ( parseResults , Math . max ( result . indentation + indentDelta , 0 ) ) + text ; } function _getTabSize ( parseResults ) { const tab = parseResults . tokenizerOutput . predominantTabSequence ; const tabLength = tab . length ; if ( tabLength === 1 && tab . charCodeAt ( 0 ) === Char . Tab ) { return defaultTabSize ; } return tabLength ; } function _createIndentationString ( parseResults , indentation ) { const tab = parseResults . tokenizerOutput . predominantTabSequence ; const tabLength = tab . length ; if ( tabLength === 1 && tab . charCodeAt ( 0 ) === Char . Tab ) { const spaceCount = indentation % defaultTabSize ; const tabCount = ( indentation - spaceCount ) / defaultTabSize ; return '\t' . repeat ( tabCount ) + ' ' . repeat ( spaceCount ) ; } return ' ' . repeat ( indentation ) ; }