From d5a71313e099dc4e336d3b198eaea60ac9e91075 Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Tue, 23 May 2023 10:27:57 +0200 Subject: [PATCH 1/9] Disable rewrites in OutlineParser --- compiler/src/dotty/tools/dotc/parsing/Parsers.scala | 6 +++--- compiler/src/dotty/tools/dotc/parsing/Scanners.scala | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 3079b26df6cd..f6c0474dafb2 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -169,9 +169,9 @@ object Parsers { } } - class Parser(source: SourceFile)(using Context) extends ParserCommon(source) { + class Parser(source: SourceFile, allowRewrite: Boolean = true)(using Context) extends ParserCommon(source) { - val in: Scanner = new Scanner(source, profile = Profile.current) + val in: Scanner = new Scanner(source, profile = Profile.current, allowRewrite = allowRewrite) // in.debugTokenStream = true // uncomment to see the token stream of the standard scanner, but not syntax highlighting /** This is the general parse entry point. @@ -4361,7 +4361,7 @@ object Parsers { /** OutlineParser parses top-level declarations in `source` to find declared classes, ignoring their bodies (which * must only have balanced braces). This is used to map class names to defining sources. */ - class OutlineParser(source: SourceFile)(using Context) extends Parser(source) with OutlineParserCommon { + class OutlineParser(source: SourceFile)(using Context) extends Parser(source, allowRewrite = false) with OutlineParserCommon { def skipBracesHook(): Option[Tree] = if (in.token == XMLSTART) Some(xmlLiteral()) else None diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index fac73bfb4992..e29b858e0978 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -170,7 +170,7 @@ object Scanners { errorButContinue(em"trailing separator is not allowed", offset + litBuf.length - 1) } - class Scanner(source: SourceFile, override val startFrom: Offset = 0, profile: Profile = NoProfile, allowIndent: Boolean = true)(using Context) extends ScannerCommon(source) { + class Scanner(source: SourceFile, override val startFrom: Offset = 0, profile: Profile = NoProfile, allowRewrite: Boolean = true, allowIndent: Boolean = true)(using Context) extends ScannerCommon(source) { val keepComments = !ctx.settings.YdropComments.value /** A switch whether operators at the start of lines can be infix operators */ @@ -179,7 +179,7 @@ object Scanners { var debugTokenStream = false val showLookAheadOnDebug = false - val rewrite = ctx.settings.rewrite.value.isDefined + val rewrite = allowRewrite && ctx.settings.rewrite.value.isDefined val oldSyntax = ctx.settings.oldSyntax.value val newSyntax = ctx.settings.newSyntax.value From c6afb380fd3dab249a898ac312cdf313c20be7f8 Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Tue, 13 Jun 2023 11:45:25 +0200 Subject: [PATCH 2/9] Patch indentation when removing braces (essentials) If the first indentation of the region is greater than the indentation of the enclosing region, we use it to indent the whole region. Otherwise we use the incremented indentation of the enclosing region. ```scala def foo = { x // we replicate indentation of x downward in region y } ``` ```scala def foo = { x // indentation of x is incorrect, we increment enclosing indentation y } ``` A bigger indentation than the required one is permitted except just after a closing brace. ```scala def bar = { x .toString // permitted indentation def foo = { } bar // must be unindented, to not fall into the body of foo } ``` And other bug fixes (see lampepfl#17522) --- .../dotty/tools/dotc/parsing/Parsers.scala | 307 +++++++++++------- .../dotty/tools/dotc/parsing/Scanners.scala | 39 ++- .../dotc/parsing/xml/MarkupParsers.scala | 5 +- .../dotty/tools/dotc/rewrites/Rewrites.scala | 14 +- tests/pos/indent-colons.scala | 5 + 5 files changed, 246 insertions(+), 124 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index f6c0474dafb2..e5534565e88d 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -27,7 +27,7 @@ import ScriptParsers._ import Decorators._ import util.Chars import scala.annotation.tailrec -import rewrites.Rewrites.{patch, overlapsPatch} +import rewrites.Rewrites.{patch, patchOver, overlapsPatch} import reporting._ import config.Feature import config.Feature.{sourceVersion, migrateTo3, globalOnlyImports} @@ -559,18 +559,21 @@ object Parsers { def inBraces[T](body: => T): T = enclosed(LBRACE, body) def inBrackets[T](body: => T): T = enclosed(LBRACKET, body) - def inBracesOrIndented[T](body: => T, rewriteWithColon: Boolean = false): T = + def inBracesOrIndented[T](body: => T, inStatSeq: Boolean = false, rewriteWithColon: Boolean = false): T = if in.token == INDENT then - val rewriteToBraces = in.rewriteNoIndent - && !testChars(in.lastOffset - 3, " =>") // braces are always optional after `=>` so none should be inserted + // braces are always optional after `=>` so none should be inserted + val afterArrow = testChars(in.lastOffset - 3, " =>") + val rewriteToBraces = in.rewriteNoIndent && !afterArrow + val rewriteToIndent = in.rewriteToIndent && !afterArrow if rewriteToBraces then indentedToBraces(body) + else if rewriteToIndent then enclosed(INDENT, toIndentedRegion(body)) else enclosed(INDENT, body) else - if in.rewriteToIndent then bracesToIndented(body, rewriteWithColon) + if in.rewriteToIndent then bracesToIndented(body, inStatSeq, rewriteWithColon) else inBraces(body) - def inDefScopeBraces[T](body: => T, rewriteWithColon: Boolean = false): T = - inBracesOrIndented(body, rewriteWithColon) + def inDefScopeBraces[T](body: => T, inStatSeq: Boolean = false, rewriteWithColon: Boolean = false): T = + inBracesOrIndented(body, inStatSeq, rewriteWithColon) /** {`,` } */ def commaSeparated[T](part: () => T): List[T] = @@ -638,20 +641,30 @@ object Parsers { /* -------- REWRITES ----------------------------------------------------------- */ - /** The last offset where a colon at the end of line would be required if a subsequent { ... } - * block would be converted to an indentation region. - */ - var possibleColonOffset: Int = -1 + object IndentRewriteState: + assert(in.rewriteToIndent) + + /** A copy of the previous token */ + var prev: TokenData = Scanners.newTokenData + + /** The last offset where a colon at the end of line would be required if a subsequent { ... } + * block would be converted to an indentation region. */ + var possibleColonOffset: Int = -1 + + /** When rewriting to indent, the minimum indent width to rewrite to */ + var minimumIndent: IndentWidth = IndentWidth.Zero + + /** When rewritting to indent, the maximum indent width to rewrite to + * to ensure an indent region is properly closed by outdentation */ + var maximumIndent: Option[IndentWidth] = None - def testChar(idx: Int, p: Char => Boolean): Boolean = { + def testChar(idx: Int, p: Char => Boolean): Boolean = val txt = source.content - idx < txt.length && p(txt(idx)) - } + idx > -1 && idx < txt.length && p(txt(idx)) - def testChar(idx: Int, c: Char): Boolean = { + def testChar(idx: Int, c: Char): Boolean = val txt = source.content - idx < txt.length && txt(idx) == c - } + idx > -1 && idx < txt.length && txt(idx) == c def testChars(from: Int, str: String): Boolean = str.isEmpty || @@ -723,30 +736,17 @@ object Parsers { t end indentedToBraces - /** The region to eliminate when replacing an opening `(` or `{` that ends a line. - * The `(` or `{` is at in.offset. - */ - def startingElimRegion(colonRequired: Boolean): (Offset, Offset) = { - val skipped = skipBlanks(in.offset + 1) - if (in.isAfterLineEnd) - if (testChar(skipped, Chars.LF) && !colonRequired) - (in.lineOffset, skipped + 1) // skip the whole line - else - (in.offset, skipped) - else if (testChar(in.offset - 1, ' ')) (in.offset - 1, in.offset + 1) - else (in.offset, in.offset + 1) - } + /** The region to eliminate when replacing a brace or parenthesis that ends a line */ + def elimRegion(offset: Offset): (Offset, Offset) = + val (start, end) = blankLinesAround(offset, offset + 1) + if testChar(end, Chars.LF) then + if testChar(start - 1, Chars.LF) then (start, end + 1) // skip the whole line + else (start, end) // skip from previous char to end of line + else (offset, end) // skip from token to next char - /** The region to eliminate when replacing a closing `)` or `}` that starts a new line - * The `)` or `}` precedes in.lastOffset. - */ - def closingElimRegion(): (Offset, Offset) = { - val skipped = skipBlanks(in.lastOffset) - if (testChar(skipped, Chars.LF)) // if `)` or `}` is on a line by itself - (source.startOfLine(in.lastOffset), skipped + 1) // skip the whole line - else // else - (in.lastOffset - 1, skipped) // move the following text up to where the `)` or `}` was - } + /** Expand the current span to its surrounding blank space */ + def blankLinesAround(start: Offset, end: Offset): (Offset, Offset) = + (skipBlanks(start - 1, -1) + 1, skipBlanks(end, 1)) /** Parse brace-enclosed `body` and rewrite it to be an indentation region instead, if possible. * If possible means: @@ -756,41 +756,114 @@ object Parsers { * 4. there is at least one token between the braces * 5. the closing brace is also at the end of the line, or it is followed by one of * `then`, `else`, `do`, `catch`, `finally`, `yield`, or `match`. - * 6. the opening brace does not follow a `=>`. The reason for this condition is that - * rewriting back to braces does not work after `=>` (since in most cases braces are omitted - * after a `=>` it would be annoying if braces were inserted). - */ - def bracesToIndented[T](body: => T, rewriteWithColon: Boolean): T = { + * 6. the opening brace does not follow a closing brace + * 7. last token is not a leading operator + * 8. not a block in a sequence of statements + * 9. cannot rewrite if colon required after a NEWLINE, e.g. + * true || + * { + * false + * } + */ + def bracesToIndented[T](body: => T, inStatSeq: Boolean, rewriteWithColon: Boolean): T = + import IndentRewriteState.* + val prevSaved = prev.saveCopy + val lastOffsetSaved = in.lastOffset val underColonSyntax = possibleColonOffset == in.lastOffset val colonRequired = rewriteWithColon || underColonSyntax - val (startOpening, endOpening) = startingElimRegion(colonRequired) - val isOutermost = in.currentRegion.isOutermost - def allBraces(r: Region): Boolean = r match { - case r: Indented => r.isOutermost || allBraces(r.enclosing) - case r: InBraces => allBraces(r.enclosing) + val (startOpening, endOpening) = elimRegion(in.offset) + def isBracesOrIndented(r: Region): Boolean = r match + case r: Indented => true + case r: InBraces => true case _ => false - } - var canRewrite = allBraces(in.currentRegion) && // test (1) - !testChars(in.lastOffset - 3, " =>") // test(6) + var canRewrite = isBracesOrIndented(in.currentRegion) && // test (1) + prevSaved.token != RBRACE && // test (6) + !(prevSaved.isOperator && prevSaved.isAfterLineEnd) && // test (7) + !inStatSeq && // test (8) + (!colonRequired || !in.isAfterLineEnd) // test (9) val t = enclosed(LBRACE, { - canRewrite &= in.isAfterLineEnd // test (2) - val curOffset = in.offset - try body - finally { - canRewrite &= in.isAfterLineEnd && in.offset != curOffset // test (3)(4) - } + if in.isAfterLineEnd && in.token != RBRACE then // test (2)(4) + toIndentedRegion: + try body + finally canRewrite &= in.isAfterLineEnd // test (3) + else + canRewrite = false + body }) - canRewrite &= (in.isAfterLineEnd || statCtdTokens.contains(in.token)) // test (5) + canRewrite &= (in.isAfterLineEnd || in.token == EOF || statCtdTokens.contains(in.token)) // test (5) if canRewrite && (!underColonSyntax || Feature.fewerBracesEnabled) then - val openingPatchStr = - if !colonRequired then "" - else if testChar(startOpening - 1, Chars.isOperatorPart(_)) then " :" - else ":" - val (startClosing, endClosing) = closingElimRegion() - patch(source, Span(startOpening, endOpening), openingPatchStr) - patch(source, Span(startClosing, endClosing), "") + val (startClosing, endClosing) = elimRegion(in.lastOffset - 1) + // patch over the added indentation to remove braces + patchOver(source, Span(startOpening, endOpening), "") + patchOver(source, Span(startClosing, endClosing), "") + if colonRequired then + if prevSaved.token == IDENTIFIER && prevSaved.isOperator then + patch(Span(prevSaved.offset, lastOffsetSaved), s"`${prevSaved.name}`:") + else if prevSaved.token == IDENTIFIER && prevSaved.name.last == '_' then + patch(Span(lastOffsetSaved), " :") + else patch(Span(lastOffsetSaved), ":") + else + // no need to force outdentation after `}` + maximumIndent = None t - } + end bracesToIndented + + /** When rewriting to indent, make sure there is an indent after a `=>\n` */ + def indentedRegionAfterArrow[T](body: => T, inCaseDef: Boolean = false): T = + if in.rewriteToIndent && (inCaseDef || in.isAfterLineEnd) then + // assert(in.last.isArrow || in.last.token == SELFARROW) + toIndentedRegion(body) + else body + + /** compute required indentation to indent region properly */ + def toIndentedRegion[T](body: => T): T = + import IndentRewriteState.* + val enclosingIndent = minimumIndent + minimumIndent = + if enclosingIndent < in.currentRegion.indentWidth then + in.currentRegion.indentWidth + else if + in.token == CASE && ( + in.currentRegion.enclosing == null || + in.currentRegion.indentWidth == in.currentRegion.enclosing.indentWidth + ) + then enclosingIndent + else enclosingIndent.increment + try body + finally + maximumIndent = Some(minimumIndent) + minimumIndent = enclosingIndent + + /** when rewriting to indent, check that indentaton is correct or patch */ + def patchIndent(): Unit = + if in.isAfterLineEnd && !in.isNewLine && in.token != OUTDENT && in.token != INDENT then + import IndentRewriteState.* + val currentIndent = in.indentWidth(in.offset) + val indentEndOffset = in.lineOffset + currentIndent.size + def isDotOrClosing = (closingParens + DOT).contains(in.token) + val needsOutdent = maximumIndent.exists: max => + currentIndent >= max || (!isDotOrClosing && currentIndent > minimumIndent) + val offByOne = + currentIndent != minimumIndent && currentIndent.isClose(minimumIndent) + if needsOutdent || !(currentIndent >= minimumIndent) || offByOne then + patch(Span(in.lineOffset, indentEndOffset), minimumIndent.toPrefix) + // no need to outdent anymore + if in.token != RBRACE then + maximumIndent = None + + def nextToken(): Unit = + if in.rewriteToIndent then + IndentRewriteState.prev = in.saveCopy + patchIndent() + in.nextToken() + + def skipToken(): Offset = + if in.rewriteToIndent then patchIndent() + in.skipToken() + + def skipToken[T](res: T): T = + if in.rewriteToIndent then patchIndent() + in.skipToken(res) /** Drop (...) or { ... }, replacing the closing element with `endStr` */ def dropParensOrBraces(start: Offset, endStr: String): Unit = { @@ -803,7 +876,7 @@ object Parsers { val preFill = if (closingStartsLine || endStr.isEmpty) "" else " " val postFill = if (in.lastOffset == in.offset) " " else "" val (startClosing, endClosing) = - if (closingStartsLine && endStr.isEmpty) closingElimRegion() + if (closingStartsLine && endStr.isEmpty) elimRegion(in.lastOffset - 1) else (in.lastOffset - 1, in.lastOffset) patch(source, Span(startClosing, endClosing), s"$preFill$endStr$postFill") } @@ -1337,7 +1410,8 @@ object Parsers { syntaxErrorOrIncomplete(em"indented definitions expected, ${in} found") def colonAtEOLOpt(): Unit = - possibleColonOffset = in.lastOffset + if in.rewriteToIndent then + IndentRewriteState.possibleColonOffset = in.lastOffset in.observeColonEOL(inTemplate = false) if in.token == COLONeol then in.nextToken() @@ -2108,7 +2182,7 @@ object Parsers { def subExpr() = subPart(expr) - def expr(location: Location): Tree = { + def expr(location: Location, inStatSeq: Boolean = false): Tree = { val start = in.offset in.token match case IMPLICIT => @@ -2136,7 +2210,7 @@ object Parsers { else new WildcardFunction(placeholderParams.reverse, t) finally placeholderParams = saved - val t = expr1(location) + val t = expr1(location, inStatSeq) if in.isArrow then placeholderParams = Nil // don't interpret `_' to the left of `=>` as placeholder wrapPlaceholders(closureRest(start, location, convertToParams(t))) @@ -2148,7 +2222,7 @@ object Parsers { wrapPlaceholders(t) } - def expr1(location: Location = Location.ElseWhere): Tree = in.token match + def expr1(location: Location = Location.ElseWhere, inStatSeq: Boolean = false): Tree = in.token match case IF => ifExpr(in.offset, If) case WHILE => @@ -2246,7 +2320,7 @@ object Parsers { case t => syntaxError(em"`inline` must be followed by an `if` or a `match`", start) t - else expr1Rest(postfixExpr(location), location) + else expr1Rest(postfixExpr(location, inStatSeq), location) end expr1 def expr1Rest(t: Tree, location: Location): Tree = @@ -2391,7 +2465,7 @@ object Parsers { in.nextToken() else accept(ARROW) - val body = + val body = indentedRegionAfterArrow: if location == Location.InBlock then block() else if location == Location.InColonArg && in.token == INDENT then blockExpr() else expr() @@ -2404,21 +2478,21 @@ object Parsers { * | InfixExpr id ColonArgument * | InfixExpr MatchClause */ - def postfixExpr(location: Location = Location.ElseWhere): Tree = - val t = postfixExprRest(prefixExpr(location), location) + def postfixExpr(location: Location = Location.ElseWhere, inStatSeq: Boolean = false): Tree = + val t = postfixExprRest(prefixExpr(location, inStatSeq), location) if location.inArgs && followingIsVararg() then Typed(t, atSpan(in.skipToken()) { Ident(tpnme.WILDCARD_STAR) }) else t def postfixExprRest(t: Tree, location: Location): Tree = - infixOps(t, in.canStartExprTokens, prefixExpr, location, ParseKind.Expr, + infixOps(t, in.canStartExprTokens, prefixExpr(_), location, ParseKind.Expr, isOperator = !(location.inArgs && followingIsVararg())) /** PrefixExpr ::= [PrefixOperator'] SimpleExpr * PrefixOperator ::= ‘-’ | ‘+’ | ‘~’ | ‘!’ (if not backquoted) */ - val prefixExpr: Location => Tree = location => + def prefixExpr(location: Location, inStatSeq: Boolean = false): Tree = if in.token == IDENTIFIER && nme.raw.isUnary(in.name) && in.canStartExprTokens.contains(in.lookahead.token) then @@ -2428,7 +2502,7 @@ object Parsers { simpleExprRest(literal(start), location, canApply = true) else atSpan(start) { PrefixOp(op, simpleExpr(location)) } - else simpleExpr(location) + else simpleExpr(location, inStatSeq) /** SimpleExpr ::= ‘new’ ConstrApp {`with` ConstrApp} [TemplateBody] * | ‘new’ TemplateBody @@ -2454,7 +2528,7 @@ object Parsers { * Quoted ::= ‘'’ ‘{’ Block ‘}’ * | ‘'’ ‘[’ Type ‘]’ */ - def simpleExpr(location: Location): Tree = { + def simpleExpr(location: Location, inStatSeq: Boolean = false): Tree = { var canApply = true val t = in.token match { case XMLSTART => @@ -2475,7 +2549,7 @@ object Parsers { atSpan(in.offset) { makeTupleOrParens(inParens(exprsInParensOrBindings())) } case LBRACE | INDENT => canApply = false - blockExpr() + blockExpr(inStatSeq) case QUOTE => atSpan(in.skipToken()) { withinStaged(StageKind.Quoted | (if (location.inPattern) StageKind.QuotedPattern else 0)) { @@ -2639,12 +2713,12 @@ object Parsers { /** BlockExpr ::= <<< (CaseClauses | Block) >>> */ - def blockExpr(): Tree = atSpan(in.offset) { + def blockExpr(inStatSeq: Boolean = false): Tree = atSpan(in.offset) { val simplify = in.token == INDENT - inDefScopeBraces { + inDefScopeBraces({ if (in.token == CASE) Match(EmptyTree, caseClauses(() => caseClause())) else block(simplify) - } + }, inStatSeq = inStatSeq) } /** Block ::= BlockStatSeq @@ -2809,14 +2883,16 @@ object Parsers { (pattern(), guard()) } CaseDef(pat, grd, atSpan(accept(ARROW)) { - if exprOnly then - if in.indentSyntax && in.isAfterLineEnd && in.token != INDENT then - warning(em"""Misleading indentation: this expression forms part of the preceding catch case. - |If this is intended, it should be indented for clarity. - |Otherwise, if the handler is intended to be empty, use a multi-line catch with - |an indented case.""") - expr() - else block() + indentedRegionAfterArrow({ + if exprOnly then + if in.indentSyntax && in.isAfterLineEnd && in.token != INDENT then + warning(em"""Misleading indentation: this expression forms part of the preceding catch case. + |If this is intended, it should be indented for clarity. + |Otherwise, if the handler is intended to be empty, use a multi-line catch with + |an indented case.""") + expr() + else block() + }, inCaseDef = true) }) } @@ -2834,7 +2910,7 @@ object Parsers { } } CaseDef(pat, EmptyTree, atSpan(accept(ARROW)) { - val t = rejectWildcardType(typ()) + val t = indentedRegionAfterArrow(rejectWildcardType(typ()), inCaseDef = true) if in.token == SEMI then in.nextToken() newLinesOptWhenFollowedBy(CASE) t @@ -4103,7 +4179,7 @@ object Parsers { Template(constr, parents, derived, self, stats) def templateBody(parents: List[Tree], rewriteWithColon: Boolean = true): (ValDef, List[Tree]) = - val r = inDefScopeBraces(templateStatSeq(), rewriteWithColon) + val r = inDefScopeBraces(templateStatSeq(), rewriteWithColon = rewriteWithColon) if in.token == WITH && parents.isEmpty then syntaxError(EarlyDefinitionsNotSupported()) in.nextToken() @@ -4211,23 +4287,28 @@ object Parsers { */ def templateStatSeq(): (ValDef, List[Tree]) = checkNoEscapingPlaceholders { val stats = new ListBuffer[Tree] + val startsAfterLineEnd = in.isAfterLineEnd val self = selfType() - while - var empty = false - if (in.token == IMPORT) - stats ++= importClause() - else if (in.token == EXPORT) - stats ++= exportClause() - else if isIdent(nme.extension) && followingIsExtension() then - stats += extension() - else if (isDefIntro(modifierTokensOrCase)) - stats +++= defOrDcl(in.offset, defAnnotsMods(modifierTokens)) - else if (isExprIntro) - stats += expr1() - else - empty = true - statSepOrEnd(stats, noPrevStat = empty) - do () + def loop = + while + var empty = false + if (in.token == IMPORT) + stats ++= importClause() + else if (in.token == EXPORT) + stats ++= exportClause() + else if isIdent(nme.extension) && followingIsExtension() then + stats += extension() + else if (isDefIntro(modifierTokensOrCase)) + stats +++= defOrDcl(in.offset, defAnnotsMods(modifierTokens)) + else if (isExprIntro) + stats += expr1(inStatSeq = true) + else + empty = true + statSepOrEnd(stats, noPrevStat = empty) + do () + if self != null && !startsAfterLineEnd then + indentedRegionAfterArrow(loop) + else loop (self, if stats.isEmpty then List(EmptyTree) else stats.toList) } @@ -4297,7 +4378,7 @@ object Parsers { if (in.token == IMPORT) stats ++= importClause() else if (isExprIntro) - stats += expr(Location.InBlock) + stats += expr(Location.InBlock, inStatSeq = true) else if in.token == IMPLICIT && !in.inModifierPosition() then stats += closure(in.offset, Location.InBlock, modifiers(BitSet(IMPLICIT))) else if isIdent(nme.extension) && followingIsExtension() then @@ -4366,7 +4447,7 @@ object Parsers { def skipBracesHook(): Option[Tree] = if (in.token == XMLSTART) Some(xmlLiteral()) else None - override def blockExpr(): Tree = { + override def blockExpr(inStatSeq: Boolean): Tree = { skipBraces() EmptyTree } diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index e29b858e0978..c50aed34c3ca 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -60,7 +60,7 @@ object Scanners { /** the base of a number */ var base: Int = 0 - def copyFrom(td: TokenData): Unit = { + def copyFrom(td: TokenData): this.type = this.token = td.token this.offset = td.offset this.lastOffset = td.lastOffset @@ -68,7 +68,9 @@ object Scanners { this.name = td.name this.strVal = td.strVal this.base = td.base - } + this + + def saveCopy: TokenData = newTokenData.copyFrom(this) def isNewLine = token == NEWLINE || token == NEWLINES def isStatSep = isNewLine || token == SEMI @@ -92,6 +94,8 @@ object Scanners { token == ARROW || token == CTXARROW } + def newTokenData: TokenData = new TokenData {} + abstract class ScannerCommon(source: SourceFile)(using Context) extends CharArrayReader with TokenData { val buf: Array[Char] = source.content def nextToken(): Unit @@ -433,7 +437,7 @@ object Scanners { // in backticks and is a binary operator. Hence, `x` is not classified as a // leading infix operator. def assumeStartsExpr(lexeme: TokenData) = - (canStartExprTokens.contains(lexeme.token) || lexeme.token == COLONeol) + (canStartExprTokens.contains(lexeme.token) || lexeme.token == COLONfollow) && (!lexeme.isOperator || nme.raw.isUnary(lexeme.name)) val lookahead = LookaheadScanner() lookahead.allowLeadingInfixOperators = false @@ -483,7 +487,7 @@ object Scanners { if (nextChar == ch) recur(idx - 1, ch, n + 1, k) else { - val k1: IndentWidth => IndentWidth = if (n == 0) k else Conc(_, Run(ch, n)) + val k1: IndentWidth => IndentWidth = if (n == 0) k else iw => k(Conc(iw, Run(ch, n))) recur(idx - 1, nextChar, 1, k1) } else recur(idx - 1, ' ', 0, identity) @@ -638,7 +642,8 @@ object Scanners { currentRegion.knownWidth = nextWidth else if (lastWidth != nextWidth) val lw = lastWidth - errorButContinue(spaceTabMismatchMsg(lw, nextWidth)) + val msg = spaceTabMismatchMsg(lw, nextWidth) + if rewriteToIndent then report.warning(msg) else errorButContinue(msg) if token != OUTDENT then handleNewIndentWidth(currentRegion, _.otherIndentWidths += nextWidth) if next.token == EMPTY then @@ -758,6 +763,8 @@ object Scanners { if endMarkerTokens.contains(lookahead.token) && source.offsetToLine(lookahead.offset) == endLine then + if rewriteToIndent && lookahead.token == MATCH then + patch(Span(offset, offset + 3), "`end`") lookahead.nextToken() if lookahead.token == EOF || source.offsetToLine(lookahead.offset) > endLine @@ -1266,6 +1273,7 @@ object Scanners { putChar(ch) ; nextRawChar() loopRest() else + next.lineOffset = if next.lastOffset < lineStartOffset then lineStartOffset else -1 finishNamedToken(IDENTIFIER, target = next) end loopRest setStrVal() @@ -1312,10 +1320,10 @@ object Scanners { } end getStringPart - private def fetchStringPart(multiLine: Boolean) = { + private def fetchStringPart(multiLine: Boolean) = offset = charOffset - 1 + lineOffset = if lastOffset < lineStartOffset then lineStartOffset else -1 getStringPart(multiLine) - } private def isTripleQuote(): Boolean = if (ch == '"') { @@ -1672,6 +1680,23 @@ object Scanners { def < (that: IndentWidth): Boolean = this <= that && !(that <= this) + def >= (that: IndentWidth): Boolean = that <= this + + def >(that: IndentWidth): Boolean = that < this + + def size: Int = this match + case Run(_, n) => n + case Conc(l, r) => l.size + r.n + + /** Add one level of indentation (one tab or two spaces depending on the last char) */ + def increment: IndentWidth = + def incRun(ch: Char, n: Int): Run = ch match + case ' ' => IndentWidth.Run(' ', n + 2) + case ch => IndentWidth.Run(ch, n + 1) + this match + case Run(ch, n) => incRun(ch, n) + case Conc(l, Run(ch, n)) => Conc(l, incRun(ch, n)) + /** Does `this` differ from `that` by not more than a single space? */ def isClose(that: IndentWidth): Boolean = this match case Run(ch1, n1) => diff --git a/compiler/src/dotty/tools/dotc/parsing/xml/MarkupParsers.scala b/compiler/src/dotty/tools/dotc/parsing/xml/MarkupParsers.scala index b3f41fab9eaa..34a179c1be01 100644 --- a/compiler/src/dotty/tools/dotc/parsing/xml/MarkupParsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/xml/MarkupParsers.scala @@ -324,8 +324,7 @@ object MarkupParsers { /** Some try/catch/finally logic used by xLiteral and xLiteralPattern. */ inline private def xLiteralCommon(f: () => Tree, ifTruncated: String => Unit): Tree = { assert(parser.in.token == Tokens.XMLSTART) - val saved = parser.in.newTokenData - saved.copyFrom(parser.in) + val saved = parser.in.saveCopy var output: Tree = null.asInstanceOf[Tree] try output = f() catch { @@ -404,7 +403,7 @@ object MarkupParsers { def escapeToScala[A](op: => A, kind: String): A = { xEmbeddedBlock = false val res = saving(parser.in.currentRegion, parser.in.currentRegion = _) { - val lbrace = parser.in.newTokenData + val lbrace = Scanners.newTokenData lbrace.token = LBRACE lbrace.offset = parser.in.charOffset - 1 lbrace.lastOffset = parser.in.lastOffset diff --git a/compiler/src/dotty/tools/dotc/rewrites/Rewrites.scala b/compiler/src/dotty/tools/dotc/rewrites/Rewrites.scala index f2dfac88d464..45cc3c4ccfe0 100644 --- a/compiler/src/dotty/tools/dotc/rewrites/Rewrites.scala +++ b/compiler/src/dotty/tools/dotc/rewrites/Rewrites.scala @@ -25,9 +25,13 @@ object Rewrites { def addPatch(span: Span, replacement: String): Unit = pbuf += Patch(span, replacement) + def patchOver(span: Span, replacement: String): Unit = + pbuf.indices.reverse.find(i => span.contains(pbuf(i).span)).foreach(pbuf.remove) + pbuf += Patch(span, replacement) + def apply(cs: Array[Char]): Array[Char] = { val delta = pbuf.map(_.delta).sum - val patches = pbuf.toList.sortBy(_.span.start) + val patches = pbuf.toList.sortBy(p => (p.span.start, p.span.end)) if (patches.nonEmpty) patches.reduceLeft {(p1, p2) => assert(p1.span.end <= p2.span.start, s"overlapping patches in $source: $p1 and $p2") @@ -71,6 +75,14 @@ object Rewrites { .addPatch(span, replacement) ) + /** Record a patch that replaces the first patch that it contains */ + def patchOver(source: SourceFile, span: Span, replacement: String)(using Context): Unit = + if ctx.reporter != Reporter.NoReporter // NoReporter is used for syntax highlighting + then ctx.settings.rewrite.value.foreach(_.patched + .getOrElseUpdate(source, new Patches(source)) + .patchOver(span, replacement) + ) + /** Patch position in `ctx.compilationUnit.source`. */ def patch(span: Span, replacement: String)(using Context): Unit = patch(ctx.compilationUnit.source, span, replacement) diff --git a/tests/pos/indent-colons.scala b/tests/pos/indent-colons.scala index eb3cbc3617ea..a48d0ffa010f 100644 --- a/tests/pos/indent-colons.scala +++ b/tests/pos/indent-colons.scala @@ -152,6 +152,11 @@ object Test23: val _ = 1 `+`: // ok x + // leading infix op + val _ = 1 + `+` : + x + val r = 1 to: 100 From de08ae0ab5f3c51e190a168f56295a45fb7bd65a Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Tue, 13 Jun 2023 11:51:41 +0200 Subject: [PATCH 3/9] Patch indentation when removing braces (cosmetic) --- .../dotty/tools/dotc/parsing/Parsers.scala | 308 +++++++++--------- .../dotty/tools/dotc/parsing/Scanners.scala | 24 +- 2 files changed, 161 insertions(+), 171 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index e5534565e88d..caa97285b97a 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -297,7 +297,7 @@ object Parsers { val offset = in.offset if in.token != token then syntaxErrorOrIncomplete(ExpectedTokenButFound(token, in.token)) - if in.token == token then in.nextToken() + if in.token == token then nextToken() offset def accept(name: Name): Int = { @@ -305,20 +305,20 @@ object Parsers { if !isIdent(name) then syntaxErrorOrIncomplete(em"`$name` expected") if isIdent(name) then - in.nextToken() + nextToken() offset } def acceptColon(): Int = val offset = in.offset - if in.isColon then { in.nextToken(); offset } + if in.isColon then { nextToken(); offset } else accept(COLONop) /** semi = nl {nl} | `;' * nl = `\n' // where allowed */ def acceptStatSep(): Unit = - if in.isNewLine then in.nextToken() else accept(SEMI) + if in.isNewLine then nextToken() else accept(SEMI) /** Parse statement separators and end markers. Ensure that there is at least * one statement separator unless the next token terminates a statement´sequence. @@ -333,7 +333,7 @@ object Parsers { def statSepOrEnd[T <: Tree](stats: ListBuffer[T], noPrevStat: Boolean = false, what: String = "statement", altEnd: Token = EOF): Boolean = def recur(sepSeen: Boolean, endSeen: Boolean): Boolean = if isStatSep then - in.nextToken() + nextToken() recur(true, endSeen) else if in.token == END then if endSeen then syntaxError(em"duplicate end marker") @@ -352,7 +352,7 @@ object Parsers { if mustStartStatTokens.contains(found) then false // it's a statement that might be legal in an outer context else - in.nextToken() // needed to ensure progress; otherwise we might cycle forever + nextToken() // needed to ensure progress; otherwise we might cycle forever skip() true @@ -589,7 +589,7 @@ object Parsers { if in.token == COMMA then val ts = new ListBuffer[T] += leading while in.token == COMMA do - in.nextToken() + nextToken() ts += part() ts.toList else leading :: Nil @@ -1102,7 +1102,7 @@ object Parsers { colonAtEOLOpt() newLineOptWhenFollowing(canStartOperand) if isColonLambda then - in.nextToken() + nextToken() recur(expr(Location.InColonArg)) else if maybePostfix && !canStartOperand(in.token) then val topInfo = opStack.head @@ -1130,19 +1130,17 @@ object Parsers { /** Accept identifier and return its name as a term name. */ def ident(): TermName = - if (isIdent) { + if isIdent then val name = in.name if name == nme.CONSTRUCTOR || name == nme.STATIC_CONSTRUCTOR then report.error( em"""Illegal backquoted identifier: `` and `` are forbidden""", in.sourcePos()) - in.nextToken() + nextToken() name - } - else { + else syntaxErrorOrIncomplete(ExpectedTokenButFound(IDENTIFIER, in.token)) nme.ERROR - } /** Accept identifier and return Ident with its name as a term name. */ def termIdent(): Ident = @@ -1176,7 +1174,7 @@ object Parsers { /** DotSelectors ::= { `.' id } */ def dotSelectors(t: Tree): Tree = - if (in.token == DOT) { in.nextToken(); dotSelectors(selector(t)) } + if (in.token == DOT) { nextToken(); dotSelectors(selector(t)) } else t private val id: Tree => Tree = x => x @@ -1189,11 +1187,11 @@ object Parsers { val start = in.offset def handleThis(qual: Ident) = - in.nextToken() + nextToken() atSpan(start) { This(qual) } def handleSuper(qual: Ident) = - in.nextToken() + nextToken() val mix = mixinQualifierOpt() val t = atSpan(start) { Super(This(qual), mix) } accept(DOT) @@ -1207,10 +1205,10 @@ object Parsers { def qual = cpy.Ident(t)(t.name.toTypeName) in.lookahead.token match case THIS => - in.nextToken() + nextToken() handleThis(qual) case SUPER => - in.nextToken() + nextToken() handleSuper(qual) case _ => t else t @@ -1245,7 +1243,7 @@ object Parsers { def simpleLiteral(): Tree = if isIdent(nme.raw.MINUS) then val start = in.offset - in.nextToken() + nextToken() literal(negOffset = start, inTypeOrSingleton = true) else literal(inTypeOrSingleton = true) @@ -1301,7 +1299,7 @@ object Parsers { syntaxErrorOrIncomplete(IllegalLiteral()) atSpan(negOffset) { Literal(Constant(null)) } } - in.nextToken() + nextToken() t } else atSpan(negOffset) { @@ -1315,7 +1313,7 @@ object Parsers { case _ => Ident(in.name) } } - in.nextToken() + nextToken() Quote(t, Nil) } else @@ -1331,11 +1329,11 @@ object Parsers { if migrateTo3 then patch(source, Span(in.offset, in.offset + 1), "Symbol(\"") patch(source, Span(in.charOffset - 1), "\")") - atSpan(in.skipToken()) { SymbolLit(in.strVal) } + atSpan(skipToken()) { SymbolLit(in.strVal) } else if (in.token == INTERPOLATIONID) interpolatedString(inPattern) else { val t = literalOf(in.token) - in.nextToken() + nextToken() t } } @@ -1348,7 +1346,7 @@ object Parsers { in.charOffset + 1 < in.buf.length && in.buf(in.charOffset) == '"' && in.buf(in.charOffset + 1) == '"' - in.nextToken() + nextToken() def nextSegment(literalOffset: Offset) = segmentBuf += Thicket( literal(literalOffset, inPattern = inPattern, inStringInterpolation = true), @@ -1356,11 +1354,11 @@ object Parsers { if (in.token == IDENTIFIER) termIdent() else if (in.token == USCORE && inPattern) { - in.nextToken() + nextToken() Ident(nme.WILDCARD) } else if (in.token == THIS) { - in.nextToken() + nextToken() This(EmptyTypeIdent) } else if (in.token == LBRACE) @@ -1385,21 +1383,21 @@ object Parsers { /* ------------- NEW LINES ------------------------------------------------- */ def newLineOpt(): Unit = - if (in.token == NEWLINE) in.nextToken() + if in.token == NEWLINE then nextToken() def newLinesOpt(): Unit = - if in.isNewLine then in.nextToken() + if in.isNewLine then nextToken() def newLineOptWhenFollowedBy(token: Int): Unit = // note: next is defined here because current == NEWLINE - if (in.token == NEWLINE && in.next.token == token) in.nextToken() + if in.token == NEWLINE && in.next.token == token then nextToken() def newLinesOptWhenFollowedBy(token: Int): Unit = - if in.isNewLine && in.next.token == token then in.nextToken() + if in.isNewLine && in.next.token == token then nextToken() def newLinesOptWhenFollowedBy(name: Name): Unit = if in.isNewLine && in.next.token == IDENTIFIER && in.next.name == name then - in.nextToken() + nextToken() def newLineOptWhenFollowing(p: Int => Boolean): Unit = // note: next is defined here because current == NEWLINE @@ -1414,13 +1412,13 @@ object Parsers { IndentRewriteState.possibleColonOffset = in.lastOffset in.observeColonEOL(inTemplate = false) if in.token == COLONeol then - in.nextToken() + nextToken() acceptIndent() def argumentStart(): Unit = colonAtEOLOpt() if migrateTo3 && in.token == NEWLINE && in.next.token == LBRACE then - in.nextToken() + nextToken() if in.indentWidth(in.offset) == in.currentRegion.indentWidth then report.errorOrMigrationWarning( em"""This opening brace will start a new statement in Scala 3. @@ -1434,7 +1432,7 @@ object Parsers { if in.token == COLONeol then if in.lookahead.token == END then in.token = NEWLINE else - in.nextToken() + nextToken() if in.token != LBRACE then acceptIndent() else newLineOptWhenFollowedBy(LBRACE) @@ -1474,11 +1472,11 @@ object Parsers { didMatch if in.token == END then - val start = in.skipToken() + val start = skipToken() if stats.isEmpty || !matchesAndSetEnd(stats.last) then syntaxError(em"misaligned end marker", Span(start, in.lastCharOffset)) in.token = IDENTIFIER // Leaving it as the original token can confuse newline insertion - in.nextToken() + nextToken() end checkEndMarker /* ------------- TYPES ------------------------------------------------------ */ @@ -1570,10 +1568,10 @@ object Parsers { imods |= Impure if token == CTXARROW then - in.nextToken() + nextToken() imods |= Given else if token == ARROW || token == TLARROW then - in.nextToken() + nextToken() else accept(ARROW) @@ -1598,16 +1596,16 @@ object Parsers { val t = if (in.token == LPAREN) { - in.nextToken() + nextToken() if (in.token == RPAREN) { - in.nextToken() + nextToken() functionRest(Nil) } else { val paramStart = in.offset def addErased() = erasedArgs.addOne(isErasedKw) - if isErasedKw then { in.skipToken(); } + if isErasedKw then { skipToken(); } addErased() val ts = in.currentRegion.withCommasExpected { funArgType() match @@ -1653,9 +1651,9 @@ object Parsers { val start = in.offset val tparams = typeParamClause(ParamOwner.TypeParam) if (in.token == TLARROW) - atSpan(start, in.skipToken())(LambdaTypeTree(tparams, toplevelTyp())) + atSpan(start, skipToken())(LambdaTypeTree(tparams, toplevelTyp())) else if (in.token == ARROW || isPureArrow(nme.PUREARROW)) { - val arrowOffset = in.skipToken() + val arrowOffset = skipToken() val body = toplevelTyp() atSpan(start, arrowOffset) { getFunction(body) match { @@ -1775,7 +1773,7 @@ object Parsers { }) else if Feature.ccEnabled && in.isIdent(nme.UPARROW) && isCaptureUpArrow then val upArrowStart = in.offset - in.nextToken() + nextToken() def cs = if in.token == LBRACE then captureSet() else atSpan(upArrowStart)(captureRoot) :: Nil @@ -1791,7 +1789,7 @@ object Parsers { def withTypeRest(t: Tree): Tree = if in.token == WITH then val withOffset = in.offset - in.nextToken() + nextToken() if in.token == LBRACE || in.token == INDENT then t else @@ -1827,12 +1825,12 @@ object Parsers { val inPattern = (staged & StageKind.QuotedPattern) != 0 val expr = if (in.name.length == 1) { - in.nextToken() + nextToken() withinStaged(StageKind.Spliced)(if (inPattern) inBraces(pattern()) else stagedBlock()) } else atSpan(in.offset + 1) { val id = Ident(in.name.drop(1)) - in.nextToken() + nextToken() id } if isType then @@ -1860,23 +1858,23 @@ object Parsers { SingletonTypeTree(simpleLiteral()) else if in.token == USCORE then if ctx.settings.YkindProjector.value == "underscores" then - val start = in.skipToken() + val start = skipToken() Ident(tpnme.USCOREkw).withSpan(Span(start, in.lastOffset, start)) else if sourceVersion.isAtLeast(future) then deprecationWarning(em"`_` is deprecated for wildcard arguments of types: use `?` instead") patch(source, Span(in.offset, in.offset + 1), "?") - val start = in.skipToken() + val start = skipToken() typeBounds().withSpan(Span(start, in.lastOffset, start)) // Allow symbols -_ and +_ through for compatibility with code written using kind-projector in Scala 3 underscore mode. // While these signify variant type parameters in Scala 2 + kind-projector, we ignore their variance markers since variance is inferred. else if (isIdent(nme.MINUS) || isIdent(nme.PLUS)) && in.lookahead.token == USCORE && ctx.settings.YkindProjector.value == "underscores" then val identName = in.name.toTypeName ++ nme.USCOREkw - val start = in.skipToken() - in.nextToken() + val start = skipToken() + nextToken() Ident(identName).withSpan(Span(start, in.lastOffset, start)) else if isIdent(nme.?) then - val start = in.skipToken() + val start = skipToken() typeBounds().withSpan(Span(start, in.lastOffset, start)) else def singletonArgs(t: Tree): Tree = @@ -1906,9 +1904,9 @@ object Parsers { else def singletonCompletion(t: Tree): Tree = if in.token == DOT then - in.nextToken() + nextToken() if in.token == TYPE then - in.nextToken() + nextToken() atSpan(startOffset(t)) { SingletonTypeTree(t) } else singletonCompletion(selector(t)) @@ -1998,7 +1996,7 @@ object Parsers { in.currentRegion.withCommasExpected { argType() match { case Ident(name) if in.token == EQUALS => - in.nextToken() + nextToken() commaSeparatedRest(NamedArg(name, argType()), () => namedTypeArg()) case firstArg => commaSeparatedRest(firstArg, () => argType()) @@ -2010,7 +2008,7 @@ object Parsers { def paramTypeOf(core: () => Tree): Tree = if in.token == ARROW || isPureArrow(nme.PUREARROW) then val isImpure = in.token == ARROW - atSpan(in.skipToken()): + atSpan(skipToken()): val tp = if isImpure then core() else capturesAndResult(core) if isImpure && Feature.pureFunsEnabled then ImpureByNameTypeTree(tp) else ByNameTypeTree(tp) @@ -2021,7 +2019,7 @@ object Parsers { if in.isIdent(nme.into) && in.featureEnabled(Feature.into) && canStartTypeTokens.contains(in.lookahead.token) - then atSpan(in.skipToken()) { Into(tp()) } + then atSpan(skipToken()) { Into(tp()) } else tp() /** FunArgType ::= Type @@ -2041,7 +2039,7 @@ object Parsers { def paramValueType(): Tree = { val t = maybeInto(toplevelTyp) if (isIdent(nme.raw.STAR)) { - in.nextToken() + nextToken() atSpan(startOffset(t)) { PostfixOp(t, Ident(tpnme.raw.STAR)) } } else t @@ -2066,7 +2064,7 @@ object Parsers { atSpan(in.offset) { TypeBoundsTree(bound(SUPERTYPE), bound(SUBTYPE)) } private def bound(tok: Int): Tree = - if (in.token == tok) { in.nextToken(); toplevelTyp() } + if (in.token == tok) { nextToken(); toplevelTyp() } else EmptyTree /** TypeParamBounds ::= TypeBounds {`<%' Type} {`:' Type} @@ -2080,21 +2078,21 @@ object Parsers { def contextBounds(pname: TypeName): List[Tree] = if in.isColon then - atSpan(in.skipToken()) { + atSpan(skipToken()) { AppliedTypeTree(toplevelTyp(), Ident(pname)) } :: contextBounds(pname) else if in.token == VIEWBOUND then report.errorOrMigrationWarning( em"view bounds `<%' are no longer supported, use a context bound `:' instead", in.sourcePos(), from = `3.0`) - atSpan(in.skipToken()) { + atSpan(skipToken()) { Function(Ident(pname) :: Nil, toplevelTyp()) } :: contextBounds(pname) else Nil def typedOpt(): Tree = - if in.isColon then { in.nextToken(); toplevelTyp() } + if in.isColon then { nextToken(); toplevelTyp() } else TypeTree().withSpan(Span(in.lastOffset)) def typeDependingOn(location: Location): Tree = @@ -2226,7 +2224,7 @@ object Parsers { case IF => ifExpr(in.offset, If) case WHILE => - atSpan(in.skipToken()) { + atSpan(skipToken()) { val cond = condExpr(DO) newLinesOpt() val body = subExpr() @@ -2237,10 +2235,10 @@ object Parsers { em"""`do while ` is no longer supported, |use `while ; do ()` instead.${rewriteNotice()}""", in.sourcePos(), from = `3.0`) - val start = in.skipToken() + val start = skipToken() atSpan(start) { val body = expr() - if (isStatSep) in.nextToken() + if (isStatSep) nextToken() val whileStart = in.offset accept(WHILE) val cond = expr() @@ -2258,12 +2256,12 @@ object Parsers { } case TRY => val tryOffset = in.offset - atSpan(in.skipToken()) { + atSpan(skipToken()) { val body = expr() val (handler, handlerStart) = if in.token == CATCH then val span = in.offset - in.nextToken() + nextToken() (if in.token == CASE then Match(EmptyTree, caseClause(exprOnly = true) :: Nil) else subExpr(), span) @@ -2281,7 +2279,7 @@ object Parsers { val finalizer = if (in.token == FINALLY) { - in.nextToken(); + nextToken(); val expr = subExpr() if expr.span.exists then expr else Literal(Constant(())) // finally without an expression @@ -2297,9 +2295,9 @@ object Parsers { ParsedTry(body, handler, finalizer) } case THROW => - atSpan(in.skipToken()) { Throw(expr()) } + atSpan(skipToken()) { Throw(expr()) } case RETURN => - atSpan(in.skipToken()) { + atSpan(skipToken()) { Return(if (isExprIntro) expr() else EmptyTree, EmptyTree) } case FOR => @@ -2309,7 +2307,7 @@ object Parsers { && !in.inModifierPosition() && in.canStartExprTokens.contains(in.lookahead.token) then - val start = in.skipToken() + val start = skipToken() in.token match case IF => ifExpr(start, InlineIf) @@ -2327,14 +2325,14 @@ object Parsers { if in.token == EQUALS then t match case Ident(_) | Select(_, _) | Apply(_, _) | PrefixOp(_, _) => - atSpan(startOffset(t), in.skipToken()) { + atSpan(startOffset(t), skipToken()) { val loc = if location.inArgs then location else Location.ElseWhere Assign(t, subPart(() => expr(loc))) } case _ => t else if in.isColon then - in.nextToken() + nextToken() ascription(t, location) else t @@ -2342,9 +2340,9 @@ object Parsers { def ascription(t: Tree, location: Location): Tree = atSpan(startOffset(t)) { in.token match { case USCORE if in.lookahead.isIdent(nme.raw.STAR) => - val uscoreStart = in.skipToken() + val uscoreStart = skipToken() val isVarargSplice = location.inArgs && followingIsVararg() - in.nextToken() + nextToken() if isVarargSplice then report.errorOrMigrationWarning( em"The syntax `x: _*` is no longer supported for vararg splices; use `x*` instead${rewriteNotice(`future-migration`)}", @@ -2377,11 +2375,11 @@ object Parsers { * `if' Expr `then' Expr [[semi] else Expr] */ def ifExpr(start: Offset, mkIf: (Tree, Tree, Tree) => If): If = - atSpan(start, in.skipToken()) { + atSpan(start, skipToken()) { val cond = condExpr(THEN) newLinesOpt() val thenp = subExpr() - val elsep = if (in.token == ELSE) { in.nextToken(); subExpr() } + val elsep = if (in.token == ELSE) { nextToken(); subExpr() } else EmptyTree mkIf(cond, thenp, elsep) } @@ -2389,7 +2387,7 @@ object Parsers { /** MatchClause ::= `match' `{' CaseClauses `}' */ def matchClause(t: Tree): Match = - atSpan(startOffset(t), in.skipToken()) { + atSpan(startOffset(t), skipToken()) { Match(t, inBracesOrIndented(caseClauses(() => caseClause()))) } @@ -2407,7 +2405,7 @@ object Parsers { */ def funParams(mods: Modifiers, location: Location): List[Tree] = if in.token == LPAREN then - in.nextToken() + nextToken() if in.token == RPAREN then Nil else @@ -2424,7 +2422,7 @@ object Parsers { em"This syntax is no longer supported; parameter needs to be enclosed in (...)${rewriteNotice(`future-migration`)}", source.atSpan(Span(start, in.lastOffset)), from = future) - in.nextToken() + nextToken() val t = infixType() if (sourceVersion == `future-migration`) { patch(source, Span(start), "(") @@ -2446,7 +2444,7 @@ object Parsers { def bindingName(): TermName = if (in.token == USCORE) { - in.nextToken() + nextToken() WildcardParamName.fresh() } else ident() @@ -2462,7 +2460,7 @@ object Parsers { if in.token == CTXARROW then if params.isEmpty then syntaxError(em"context function literals require at least one formal parameter", Span(start, in.lastOffset)) - in.nextToken() + nextToken() else accept(ARROW) val body = indentedRegionAfterArrow: @@ -2481,7 +2479,7 @@ object Parsers { def postfixExpr(location: Location = Location.ElseWhere, inStatSeq: Boolean = false): Tree = val t = postfixExprRest(prefixExpr(location, inStatSeq), location) if location.inArgs && followingIsVararg() then - Typed(t, atSpan(in.skipToken()) { Ident(tpnme.WILDCARD_STAR) }) + Typed(t, atSpan(skipToken()) { Ident(tpnme.WILDCARD_STAR) }) else t @@ -2539,7 +2537,7 @@ object Parsers { case BACKQUOTED_IDENT | THIS | SUPER => simpleRef() case USCORE => - val start = in.skipToken() + val start = skipToken() val pname = WildcardParamName.fresh() val param = ValDef(pname, TypeTree(), EmptyTree).withFlags(SyntheticTermParam) .withSpan(Span(start)) @@ -2551,7 +2549,7 @@ object Parsers { canApply = false blockExpr(inStatSeq) case QUOTE => - atSpan(in.skipToken()) { + atSpan(skipToken()) { withinStaged(StageKind.Quoted | (if (location.inPattern) StageKind.QuotedPattern else 0)) { val body = if (in.token == LBRACKET) inBrackets(typ()) @@ -2563,14 +2561,14 @@ object Parsers { canApply = false newExpr() case MACRO => - val start = in.skipToken() + val start = skipToken() MacroTree(simpleExpr(Location.ElseWhere)) case _ => if isLiteral then literal() else if in.isColon then syntaxError(IllegalStartSimpleExpr(tokenString(in.token))) - in.nextToken() + nextToken() simpleExpr(location) else val start = in.lastOffset @@ -2584,7 +2582,7 @@ object Parsers { if (canApply) argumentStart() in.token match case DOT => - in.nextToken() + nextToken() simpleExprRest(selectorOrMatch(t), location, canApply = true) case LBRACKET => val tapp = atSpan(startOffset(t), in.offset) { TypeApply(t, typeArgs(namedOK = true, wildOK = false)) } @@ -2593,7 +2591,7 @@ object Parsers { val app = atSpan(startOffset(t), in.offset) { mkApply(t, argumentExprs()) } simpleExprRest(app, location, canApply = true) case USCORE => - atSpan(startOffset(t), in.skipToken()) { PostfixOp(t, Ident(nme.WILDCARD)) } + atSpan(startOffset(t), skipToken()) { PostfixOp(t, Ident(nme.WILDCARD)) } case _ => if in.isColon && location == Location.InParens && followingIsLambdaParams() then t match @@ -2606,7 +2604,7 @@ object Parsers { } case _ => t else if isColonLambda then - val app = atSpan(startOffset(t), in.skipToken()) { + val app = atSpan(startOffset(t), skipToken()) { Apply(t, expr(Location.InColonArg) :: Nil) } simpleExprRest(app, location, canApply = true) @@ -2617,7 +2615,7 @@ object Parsers { * | ‘new’ TemplateBody */ def newExpr(): Tree = - val start = in.skipToken() + val start = skipToken() def reposition(t: Tree) = t.withSpan(Span(start, in.lastOffset)) possibleTemplateStart() val parents = @@ -2655,7 +2653,7 @@ object Parsers { if in.token == RPAREN then (Nil, false) else if isIdent(nme.using) then - in.nextToken() + nextToken() (commaSeparated(argumentExpr), true) else (commaSeparated(argumentExpr), false) @@ -2739,7 +2737,7 @@ object Parsers { /** Guard ::= if PostfixExpr */ def guard(): Tree = - if (in.token == IF) { in.nextToken(); postfixExpr(Location.InGuard) } + if (in.token == IF) { nextToken(); postfixExpr(Location.InGuard) } else EmptyTree /** Enumerators ::= Generator {semi Enumerator | Guard} @@ -2748,7 +2746,7 @@ object Parsers { def enumeratorsRest(): List[Tree] = if (isStatSep) { - in.nextToken() + nextToken() if (in.token == DO || in.token == YIELD || in.token == RBRACE) Nil else enumerator() :: enumeratorsRest() } @@ -2765,14 +2763,14 @@ object Parsers { else if (in.token == CASE) generator() else { val pat = pattern1() - if (in.token == EQUALS) atSpan(startOffset(pat), in.skipToken()) { GenAlias(pat, subExpr()) } + if (in.token == EQUALS) atSpan(startOffset(pat), skipToken()) { GenAlias(pat, subExpr()) } else generatorRest(pat, casePat = false) } /** Generator ::= [‘case’] Pattern `<-' Expr */ def generator(): Tree = { - val casePat = if (in.token == CASE) { in.nextToken(); true } else false + val casePat = if (in.token == CASE) { nextToken(); true } else false generatorRest(pattern1(), casePat) } @@ -2791,14 +2789,14 @@ object Parsers { * | ‘for’ Enumerators (‘do‘ | ‘yield’) Expr */ def forExpr(): Tree = - atSpan(in.skipToken()) { + atSpan(skipToken()) { var wrappedEnums = true val start = in.offset val forEnd = in.lastOffset val leading = in.token val enums = if (leading == LBRACE || leading == LPAREN && followingIsEnclosedGenerators()) { - in.nextToken() + nextToken() val res = if (leading == LBRACE || in.token == CASE) enumerators() @@ -2850,12 +2848,12 @@ object Parsers { } newLinesOpt() if (in.token == YIELD) { - in.nextToken() + nextToken() ForYield(enums, subExpr()) } else if (in.token == DO) { if (rewriteToOldSyntax()) dropTerminator() - in.nextToken() + nextToken() ForDo(enums, subExpr()) } else { @@ -2903,7 +2901,7 @@ object Parsers { accept(CASE) in.token match { case USCORE if in.lookahead.isArrow => - val start = in.skipToken() + val start = skipToken() Ident(tpnme.WILDCARD).withSpan(Span(start, in.lastOffset, start)) case _ => rejectWildcardType(infixType()) @@ -2911,7 +2909,7 @@ object Parsers { } CaseDef(pat, EmptyTree, atSpan(accept(ARROW)) { val t = indentedRegionAfterArrow(rejectWildcardType(typ()), inCaseDef = true) - if in.token == SEMI then in.nextToken() + if in.token == SEMI then nextToken() newLinesOptWhenFollowedBy(CASE) t }) @@ -2928,7 +2926,7 @@ object Parsers { else pat def patternAlts(location: Location): List[Tree] = - if (isIdent(nme.raw.BAR)) { in.nextToken(); pattern1(location) :: patternAlts(location) } + if (isIdent(nme.raw.BAR)) { nextToken(); pattern1(location) :: patternAlts(location) } else Nil /** Pattern1 ::= PatVar Ascription @@ -2950,7 +2948,7 @@ object Parsers { warnFrom = `3.3`, errorFrom = future ) - in.nextToken() + nextToken() ascription(p, location) else p @@ -2960,7 +2958,7 @@ object Parsers { def pattern3(): Tree = val p = infixPattern() if followingIsVararg() then - val start = in.skipToken() + val start = skipToken() p match case p @ Ident(name) if name.isVarPattern => Typed(p, atSpan(start) { Ident(tpnme.WILDCARD_STAR) }) @@ -2973,7 +2971,7 @@ object Parsers { */ val pattern2: () => Tree = () => pattern3() match case p @ Ident(name) if in.token == AT => - val offset = in.skipToken() + val offset = skipToken() pattern3() match { case pt @ Bind(nme.WILDCARD, pt1: Typed) if pt.mods.is(Given) => atSpan(startOffset(p), 0) { Bind(name, pt1).withMods(pt.mods) } @@ -3024,7 +3022,7 @@ object Parsers { xmlLiteralPattern() case GIVEN => atSpan(in.offset) { - val givenMod = atSpan(in.skipToken())(Mod.Given()) + val givenMod = atSpan(skipToken())(Mod.Given()) val typed = Typed(Ident(nme.WILDCARD), refinedType()) Bind(nme.WILDCARD, typed).withMods(addMod(Modifiers(), givenMod)) } @@ -3041,7 +3039,7 @@ object Parsers { def simplePatternRest(t: Tree): Tree = if in.token == DOT then - in.nextToken() + nextToken() simplePatternRest(selector(t)) else var p = t @@ -3102,7 +3100,7 @@ object Parsers { private def addModifier(mods: Modifiers): Modifiers = { val tok = in.token val name = in.name - val mod = atSpan(in.skipToken()) { modOfToken(tok, name) } + val mod = atSpan(skipToken()) { modOfToken(tok, name) } if (mods.isOneOf(mod.flags)) syntaxError(RepeatedModifier(mod.flags.flagsString)) addMod(mods, mod) @@ -3127,7 +3125,7 @@ object Parsers { if sourceVersion.isAtLeast(future) then deprecationWarning( em"The [this] qualifier will be deprecated in the future; it should be dropped.") - in.nextToken() + nextToken() mods | Local else mods.withPrivateWithin(ident().toTypeName) } @@ -3156,7 +3154,7 @@ object Parsers { val mods1 = addModifier(mods) loop(if (isAccessMod) accessQualifierOpt(mods1) else mods1) else if (in.isNewLine && (mods.hasFlags || mods.hasAnnotations)) { - in.nextToken() + nextToken() loop(mods) } else @@ -3256,7 +3254,7 @@ object Parsers { def checkVarianceOK(): Boolean = val ok = ownerKind != ParamOwner.Def && ownerKind != ParamOwner.TypeParam if !ok then syntaxError(em"no `+/-` variance annotation allowed here") - in.nextToken() + nextToken() ok def typeParam(): TypeDef = { @@ -3267,7 +3265,7 @@ object Parsers { if Feature.ccEnabled && in.token == SEALED then if ownerKind == ParamOwner.Def then mods |= Sealed else syntaxError(em"`sealed` modifier only allowed for method type parameters") - in.nextToken() + nextToken() if isIdent(nme.raw.PLUS) && checkVarianceOK() then mods |= Covariant else if isIdent(nme.raw.MINUS) && checkVarianceOK() then @@ -3275,7 +3273,7 @@ object Parsers { atSpan(start, nameStart) { val name = if (isAbstractOwner && in.token == USCORE) { - in.nextToken() + nextToken() WildcardParamName.fresh().toTypeName } else ident().toTypeName @@ -3327,7 +3325,7 @@ object Parsers { ): List[ValDef] = { var impliedMods: Modifiers = EmptyModifiers - def addParamMod(mod: () => Mod) = impliedMods = addMod(impliedMods, atSpan(in.skipToken()) { mod() }) + def addParamMod(mod: () => Mod) = impliedMods = addMod(impliedMods, atSpan(skipToken()) { mod() }) def paramMods() = if in.token == IMPLICIT then @@ -3345,10 +3343,10 @@ object Parsers { mods = addFlag(modifiers(start = mods), ParamAccessor) mods = if in.token == VAL then - in.nextToken() + nextToken() mods else if in.token == VAR then - val mod = atSpan(in.skipToken()) { Mod.Var() } + val mod = atSpan(skipToken()) { Mod.Var() } addMod(mods, mod) else if (!(mods.flags &~ (ParamAccessor | Inline | Erased | impliedMods.flags)).isEmpty) @@ -3369,7 +3367,7 @@ object Parsers { // needed?, it's checked later anyway val tpt = paramType() val default = - if (in.token == EQUALS) { in.nextToken(); subExpr() } + if (in.token == EQUALS) { nextToken(); subExpr() } else EmptyTree if (impliedMods.mods.nonEmpty) impliedMods = impliedMods.withMods(Nil) // keep only flags, so that parameter positions don't overlap @@ -3529,12 +3527,12 @@ object Parsers { in.sourcePos(), from = future) patch(source, Span(in.offset, in.offset + 1), "*") - ImportSelector(atSpan(in.skipToken()) { Ident(nme.WILDCARD) }) + ImportSelector(atSpan(skipToken()) { Ident(nme.WILDCARD) }) /** 'given [InfixType]' */ def givenSelector() = ImportSelector( - atSpan(in.skipToken()) { Ident(nme.EMPTY) }, + atSpan(skipToken()) { Ident(nme.EMPTY) }, bound = if canStartInfixTypeTokens.contains(in.token) then rejectWildcardType(infixType()) else EmptyTree) @@ -3550,7 +3548,7 @@ object Parsers { patch(source, Span(in.offset, in.offset + 2), if testChar(in.offset - 1, ' ') && testChar(in.offset + 2, ' ') then "as" else " as ") - atSpan(startOffset(from), in.skipToken()) { + atSpan(startOffset(from), skipToken()) { val to = if in.token == USCORE then wildcardIdent() else termIdent() ImportSelector(from, if to.name == nme.ERROR then EmptyTree else to) } @@ -3621,16 +3619,16 @@ object Parsers { */ def defOrDcl(start: Int, mods: Modifiers): Tree = in.token match { case VAL => - in.nextToken() + nextToken() patDefOrDcl(start, mods) case VAR => - val mod = atSpan(in.skipToken()) { Mod.Var() } + val mod = atSpan(skipToken()) { Mod.Var() } val mod1 = addMod(mods, mod) patDefOrDcl(start, mod1) case DEF => - defDefOrDcl(start, in.skipToken(mods)) + defDefOrDcl(start, skipToken(mods)) case TYPE => - typeDefOrDcl(start, in.skipToken(mods)) + typeDefOrDcl(start, skipToken(mods)) case CASE if inEnum => enumCase(start, mods) case _ => @@ -3648,7 +3646,7 @@ object Parsers { val first = pattern2() var lhs = first match { case id: Ident if in.token == COMMA => - in.nextToken() + nextToken() id :: commaSeparated(() => termIdent()) case _ => first :: Nil @@ -3715,7 +3713,7 @@ object Parsers { false if (in.token == THIS) { - in.nextToken() + nextToken() val vparamss = termParamClauses(numLeadParams = numLeadParams) if (vparamss.isEmpty || vparamss.head.take(1).exists(_.mods.isOneOf(GivenOrImplicit))) in.token match { @@ -3749,7 +3747,7 @@ object Parsers { if (migrateTo3) newLineOptWhenFollowedBy(LBRACE) val rhs = if in.token == EQUALS then - in.nextToken() + nextToken() subExpr() else if !tpt.isEmpty then EmptyTree @@ -3776,7 +3774,7 @@ object Parsers { atSpan(in.offset) { inBracesOrIndented { val stats = selfInvocation() :: ( - if (isStatSep) { in.nextToken(); blockStatSeq() } + if (isStatSep) { nextToken(); blockStatSeq() } else Nil) Block(stats, Literal(Constant(()))) } @@ -3808,12 +3806,12 @@ object Parsers { } in.token match { case EQUALS => - in.nextToken() + nextToken() makeTypeDef(toplevelTyp()) case SUBTYPE | SUPERTYPE => val bounds = typeBounds() if (in.token == EQUALS) { - val eqOffset = in.skipToken() + val eqOffset = skipToken() var rhs = toplevelTyp() rhs match { case mtt: MatchTypeTree => @@ -3849,19 +3847,19 @@ object Parsers { def tmplDef(start: Int, mods: Modifiers): Tree = in.token match { case TRAIT => - classDef(start, in.skipToken(addFlag(mods, Trait))) + classDef(start, skipToken(addFlag(mods, Trait))) case CLASS => - classDef(start, in.skipToken(mods)) + classDef(start, skipToken(mods)) case CASECLASS => - classDef(start, in.skipToken(mods | Case)) + classDef(start, skipToken(mods | Case)) case OBJECT => - objectDef(start, in.skipToken(mods | Module)) + objectDef(start, skipToken(mods | Module)) case CASEOBJECT => - objectDef(start, in.skipToken(mods | Case | Module)) + objectDef(start, skipToken(mods | Case | Module)) case ENUM => - enumDef(start, in.skipToken(mods | Enum)) + enumDef(start, skipToken(mods | Enum)) case GIVEN => - givenDef(start, mods, atSpan(in.skipToken()) { Mod.Given() }) + givenDef(start, mods, atSpan(skipToken()) { Mod.Given() }) case _ => val start = in.lastOffset syntaxErrorOrIncomplete(ExpectedStartOfTopLevelDefinition()) @@ -3931,7 +3929,7 @@ object Parsers { atSpan(start, nameStart) { val id = termIdent() if (in.token == COMMA) { - in.nextToken() + nextToken() val ids = commaSeparated(() => termIdent()) PatDef(mods1, id :: ids, TypeTree(), EmptyTree) } @@ -3953,7 +3951,7 @@ object Parsers { def caseTemplate(constr: DefDef): Template = { val parents = if (in.token == EXTENDS) { - in.nextToken() + nextToken() constrApps() } else Nil @@ -4029,7 +4027,7 @@ object Parsers { * {UsingParamClause} ExtMethods */ def extension(): ExtMethods = - val start = in.skipToken() + val start = skipToken() val tparams = typeParamClauseOpt(ParamOwner.Def) val leadParamss = ListBuffer[List[ValDef]]() def numLeadParams = leadParamss.map(_.length).sum @@ -4041,7 +4039,7 @@ object Parsers { leadParamss ++= termParamClauses(givenOnly = true, numLeadParams = numLeadParams) if in.isColon then syntaxError(em"no `:` expected here") - in.nextToken() + nextToken() val methods: List[Tree] = if in.token == EXPORT then exportClause() @@ -4105,7 +4103,7 @@ object Parsers { val ts = val tok = in.token if (tok == WITH || tok == COMMA) && tok != exclude then - in.nextToken() + nextToken() constrApps(exclude = if tok == WITH then COMMA else WITH) else Nil t :: ts @@ -4117,7 +4115,7 @@ object Parsers { val la = in.lookahead la.isAfterLineEnd || la.token == LBRACE if in.token == WITH && !isTemplateStart then - in.nextToken() + nextToken() constrApp() :: withConstrApps() else Nil @@ -4127,7 +4125,7 @@ object Parsers { def template(constr: DefDef, isEnum: Boolean = false): Template = { val parents = if (in.token == EXTENDS) { - in.nextToken() + nextToken() if (in.token == LBRACE || in.token == COLONeol) { report.errorOrMigrationWarning( em"`extends` must be followed by at least one parent", @@ -4140,7 +4138,7 @@ object Parsers { newLinesOptWhenFollowedBy(nme.derives) val derived = if (isIdent(nme.derives)) { - in.nextToken() + nextToken() commaSeparated(() => convertToTypeId(qualId())) } else Nil @@ -4182,7 +4180,7 @@ object Parsers { val r = inDefScopeBraces(templateStatSeq(), rewriteWithColon = rewriteWithColon) if in.token == WITH && parents.isEmpty then syntaxError(EarlyDefinitionsNotSupported()) - in.nextToken() + nextToken() template(emptyConstructor) r @@ -4222,9 +4220,9 @@ object Parsers { while var empty = false if (in.token == PACKAGE) { - val start = in.skipToken() + val start = skipToken() if (in.token == OBJECT) { - in.nextToken() + nextToken() stats += objectDef(start, Modifiers(Package)) } else stats += packaging(start) @@ -4255,19 +4253,19 @@ object Parsers { atSpan(in.offset) { val selfName = if in.token == THIS then - in.nextToken() + nextToken() nme.WILDCARD else ident() val selfTpt = if in.isColon then - in.nextToken() + nextToken() infixType() else if selfName == nme.WILDCARD then accept(COLONfollow) TypeTree() if in.token == ARROW then in.token = SELFARROW // suppresses INDENT insertion after `=>` - in.nextToken() + nextToken() else syntaxError(em"`=>` expected after self type") makeSelfDef(selfName, selfTpt) @@ -4397,12 +4395,12 @@ object Parsers { def compilationUnit(): Tree = checkNoEscapingPlaceholders { def topstats(): List[Tree] = { val ts = new ListBuffer[Tree] - while (in.token == SEMI) in.nextToken() + while (in.token == SEMI) nextToken() val start = in.offset if (in.token == PACKAGE) { - in.nextToken() + nextToken() if (in.token == OBJECT) { - in.nextToken() + nextToken() ts += objectDef(start, Modifiers(Package)) if (in.token != EOF) { statSepOrEnd(ts, what = "toplevel definition") diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index c50aed34c3ca..bde7dc146338 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -88,7 +88,7 @@ object Scanners { def isOperator = token == BACKQUOTED_IDENT - || token == IDENTIFIER && isOperatorPart(name(name.length - 1)) + || token == IDENTIFIER && isOperatorPart(name.last) def isArrow = token == ARROW || token == CTXARROW @@ -268,11 +268,10 @@ object Scanners { if (idx >= 0 && idx <= lastKeywordStart) handleMigration(kwArray(idx)) else IDENTIFIER - def newTokenData: TokenData = new TokenData {} - /** We need one token lookahead and one token history */ val next = newTokenData + val last = newTokenData private val prev = newTokenData /** The current region. This is initially an Indented region with zero indentation width. */ @@ -527,7 +526,7 @@ object Scanners { * * The following tokens can start an indentation region: * - * : = => <- if then else while do try catch + * : = => <- if then else while do try catch * finally for yield match throw return with * * Inserting an INDENT starts a new indentation region with the indentation of the current @@ -1665,18 +1664,11 @@ object Scanners { case Run(ch: Char, n: Int) case Conc(l: IndentWidth, r: Run) - def <= (that: IndentWidth): Boolean = this match { - case Run(ch1, n1) => - that match { - case Run(ch2, n2) => n1 <= n2 && (ch1 == ch2 || n1 == 0) - case Conc(l, r) => this <= l - } - case Conc(l1, r1) => - that match { - case Conc(l2, r2) => l1 == l2 && r1 <= r2 - case _ => false - } - } + def <= (that: IndentWidth): Boolean = (this, that) match + case (Run(ch1, n1), Run(ch2, n2)) => n1 <= n2 && (ch1 == ch2 || n1 == 0) + case (Conc(l1, r1), Conc(l2, r2)) => (l1 == l2 && r1 <= r2) || this <= l2 + case (_, Conc(l2, _)) => this <= l2 + case _ => false def < (that: IndentWidth): Boolean = this <= that && !(that <= this) From 5657b69391df2cdfba37afd9d0b3d4fd92136ca2 Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Tue, 13 Jun 2023 14:05:22 +0200 Subject: [PATCH 4/9] Add indent rewrite tests --- .../dotty/tools/dotc/parsing/Scanners.scala | 2 - .../dotty/tools/dotc/CompilationTests.scala | 11 +- .../tools/vulpix/TestConfiguration.scala | 1 + tests/rewrites/indent-3-spaces.check | 21 ++ tests/rewrites/indent-3-spaces.scala | 26 ++ tests/rewrites/indent-comments.check | 25 ++ tests/rewrites/indent-comments.scala | 27 ++ tests/rewrites/indent-mix-brace.check | 17 ++ tests/rewrites/indent-mix-brace.scala | 21 ++ tests/rewrites/indent-mix-tab-space.check | 22 ++ tests/rewrites/indent-mix-tab-space.scala | 27 ++ tests/rewrites/indent-rewrite.check | 241 +++++++++++++++ tests/rewrites/indent-rewrite.scala | 274 ++++++++++++++++++ 13 files changed, 710 insertions(+), 5 deletions(-) create mode 100644 tests/rewrites/indent-3-spaces.check create mode 100644 tests/rewrites/indent-3-spaces.scala create mode 100644 tests/rewrites/indent-comments.check create mode 100644 tests/rewrites/indent-comments.scala create mode 100644 tests/rewrites/indent-mix-brace.check create mode 100644 tests/rewrites/indent-mix-brace.scala create mode 100644 tests/rewrites/indent-mix-tab-space.check create mode 100644 tests/rewrites/indent-mix-tab-space.scala create mode 100644 tests/rewrites/indent-rewrite.check create mode 100644 tests/rewrites/indent-rewrite.scala diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index bde7dc146338..ea3960839f06 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -762,8 +762,6 @@ object Scanners { if endMarkerTokens.contains(lookahead.token) && source.offsetToLine(lookahead.offset) == endLine then - if rewriteToIndent && lookahead.token == MATCH then - patch(Span(offset, offset + 3), "`end`") lookahead.nextToken() if lookahead.token == EOF || source.offsetToLine(lookahead.offset) > endLine diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index 4e86a3b83383..7a36de330723 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -82,9 +82,14 @@ class CompilationTests { compileFile("tests/rewrites/rewrites3x.scala", defaultOptions.and("-rewrite", "-source", "future-migration")), compileFile("tests/rewrites/filtering-fors.scala", defaultOptions.and("-rewrite", "-source", "3.2-migration")), compileFile("tests/rewrites/refutable-pattern-bindings.scala", defaultOptions.and("-rewrite", "-source", "3.2-migration")), - compileFile("tests/rewrites/i8982.scala", defaultOptions.and("-indent", "-rewrite")), - compileFile("tests/rewrites/i9632.scala", defaultOptions.and("-indent", "-rewrite")), - compileFile("tests/rewrites/i11895.scala", defaultOptions.and("-indent", "-rewrite")), + compileFile("tests/rewrites/i8982.scala", indentRewrite), + compileFile("tests/rewrites/i9632.scala", indentRewrite), + compileFile("tests/rewrites/i11895.scala", indentRewrite), + compileFile("tests/rewrites/indent-rewrite.scala", indentRewrite), + compileFile("tests/rewrites/indent-comments.scala", indentRewrite), + compileFile("tests/rewrites/indent-mix-tab-space.scala", indentRewrite), + compileFile("tests/rewrites/indent-3-spaces.scala", indentRewrite), + compileFile("tests/rewrites/indent-mix-brace.scala", indentRewrite), compileFile("tests/rewrites/i12340.scala", unindentOptions.and("-rewrite")), compileFile("tests/rewrites/i17187.scala", unindentOptions.and("-rewrite")), compileFile("tests/rewrites/i17399.scala", unindentOptions.and("-rewrite")), diff --git a/compiler/test/dotty/tools/vulpix/TestConfiguration.scala b/compiler/test/dotty/tools/vulpix/TestConfiguration.scala index 5d2992b50a09..170454ac3347 100644 --- a/compiler/test/dotty/tools/vulpix/TestConfiguration.scala +++ b/compiler/test/dotty/tools/vulpix/TestConfiguration.scala @@ -65,6 +65,7 @@ object TestConfiguration { val commonOptions = Array("-indent") ++ checkOptions ++ noCheckOptions ++ yCheckOptions val defaultOptions = TestFlags(basicClasspath, commonOptions) + val indentRewrite = defaultOptions.and("-rewrite") val unindentOptions = TestFlags(basicClasspath, Array("-no-indent") ++ checkOptions ++ noCheckOptions ++ yCheckOptions) val withCompilerOptions = defaultOptions.withClasspath(withCompilerClasspath).withRunClasspath(withCompilerClasspath) diff --git a/tests/rewrites/indent-3-spaces.check b/tests/rewrites/indent-3-spaces.check new file mode 100644 index 000000000000..a9dd85872ed4 --- /dev/null +++ b/tests/rewrites/indent-3-spaces.check @@ -0,0 +1,21 @@ +// Rewrite to indent, keeping 3 spaces as indentation + +def m1 = + def m2 = + "" + + "" + + "" + m2 + +def m4 = + def m5 = + def m6 = + val x = "" + x + .apply(0) + .toString + m6 + .toString + m5 + + m5 + .toString diff --git a/tests/rewrites/indent-3-spaces.scala b/tests/rewrites/indent-3-spaces.scala new file mode 100644 index 000000000000..9df6e253f074 --- /dev/null +++ b/tests/rewrites/indent-3-spaces.scala @@ -0,0 +1,26 @@ +// Rewrite to indent, keeping 3 spaces as indentation + +def m1 = { + def m2 = { + "" + + "" + + "" + } + m2 +} + +def m4 = { + def m5 = { + def m6 = { + val x = "" + x + .apply(0) + .toString + } + m6 + .toString + } + m5 + + m5 + .toString +} diff --git a/tests/rewrites/indent-comments.check b/tests/rewrites/indent-comments.check new file mode 100644 index 000000000000..38290f23e8fa --- /dev/null +++ b/tests/rewrites/indent-comments.check @@ -0,0 +1,25 @@ +// Rewriting to indent should preserve comments +class A: /* 1 */ /* 2 */ + def m1(b: Boolean) = /* 3 */ /* 4 */ + val x = if (b) + /* 5 */ + "true" + /* 6 */ + else + /* 7 */ + "false" + /* 8 */ + /* 9 */ x.toBoolean + /* 10 */ /* 11 */ + /* 12 */def m2 = // 12 + m1:// 14 + /* 15 */// 16 + true + /* 17 */// 18 +// because of the missing indent before { +// the scanner inserts a new line between || and { +// cannot rewrite to indentation without messing the comments up + true ||// 19 + /* 20 */{ + false + }// 21 diff --git a/tests/rewrites/indent-comments.scala b/tests/rewrites/indent-comments.scala new file mode 100644 index 000000000000..87bc8bda33d6 --- /dev/null +++ b/tests/rewrites/indent-comments.scala @@ -0,0 +1,27 @@ +// Rewriting to indent should preserve comments +class A /* 1 */ { /* 2 */ + def m1(b: Boolean) = /* 3 */ { /* 4 */ + val x = if (b) + /* 5 */ { + "true" + } /* 6 */ + else + { /* 7 */ + "false" + /* 8 */ } +/* 9 */ x.toBoolean + /* 10 */ } /* 11 */ +/* 12 */def m2 = {// 12 +m1// 14 + /* 15 */{// 16 +true +/* 17 */}// 18 +// because of the missing indent before { +// the scanner inserts a new line between || and { +// cannot rewrite to indentation without messing the comments up +true ||// 19 +/* 20 */{ + false +}// 21 +} +} diff --git a/tests/rewrites/indent-mix-brace.check b/tests/rewrites/indent-mix-brace.check new file mode 100644 index 000000000000..eb4752e1cb2b --- /dev/null +++ b/tests/rewrites/indent-mix-brace.check @@ -0,0 +1,17 @@ +// A mix of nested in-brace regions and indented regions + +class A: + def m1 = + "" + + def m2 = + def m3 = + val x = "" + x + m3 + +class B: + def foo = + def bar = + "" + bar diff --git a/tests/rewrites/indent-mix-brace.scala b/tests/rewrites/indent-mix-brace.scala new file mode 100644 index 000000000000..944537fc341f --- /dev/null +++ b/tests/rewrites/indent-mix-brace.scala @@ -0,0 +1,21 @@ +// A mix of nested in-brace regions and indented regions + +class A: + def m1 = { + "" + } + + def m2 = { +def m3 = + val x = "" + x +m3 + } + +class B { + def foo = + def bar = { + "" + } + bar +} diff --git a/tests/rewrites/indent-mix-tab-space.check b/tests/rewrites/indent-mix-tab-space.check new file mode 100644 index 000000000000..4f25839ccfda --- /dev/null +++ b/tests/rewrites/indent-mix-tab-space.check @@ -0,0 +1,22 @@ +// Contains an ugly but valid mix of spaces and tabs +// Rewrite to significant indentation syntax + +def m1 = + def m2 = + "" + + "" + + "" + m2 + +def m4 = + def m5 = + def m6 = + val x = "" + x + .apply(0) + .toString + m6 + .toString + m5 + + m5 + .toString diff --git a/tests/rewrites/indent-mix-tab-space.scala b/tests/rewrites/indent-mix-tab-space.scala new file mode 100644 index 000000000000..4a77fd1cbde6 --- /dev/null +++ b/tests/rewrites/indent-mix-tab-space.scala @@ -0,0 +1,27 @@ +// Contains an ugly but valid mix of spaces and tabs +// Rewrite to significant indentation syntax + +def m1 = { + def m2 = { + "" + + "" + + "" + } + m2 +} + +def m4 = { + def m5 = { + def m6 = { + val x = "" + x + .apply(0) + .toString + } + m6 + .toString + } + m5 + + m5 + .toString +} diff --git a/tests/rewrites/indent-rewrite.check b/tests/rewrites/indent-rewrite.check new file mode 100644 index 000000000000..c18b123f2af7 --- /dev/null +++ b/tests/rewrites/indent-rewrite.check @@ -0,0 +1,241 @@ +// A collection of patterns found when rewriting the community build to indent + +trait C1: + + class CC1 +// do not remove braces if empty region + class CC2 { + + } +// do not remove braces if open brace is not followed by new line + def m1(x: Int) = + { x + .toString + } +// add indent to pass an argument (fewer braces) + def m2: String = + m1: + 5 +// indent inner method + def m3: Int = + def seq = + Seq( + "1", + "2" + ) + seq + (1) + .toInt +// indent refinement + def m4: Any: + def foo: String + = + new: + def foo: String = + """ +Hello, World! +""" +// indent end marker + end m4 + +// fix off-by-one indentation + val x = "" + + def m5(x: String): String = + def inner: Boolean = + true + x + + // unindent properly when needed + def m6(xs: Seq[String]): String = + xs + .map: + x => x + .filter: + x => x.size > 0 + println("foo") + + def foo: String = + "" + foo + +// do not remove braces if closing braces not followed by new line + def m7: String = { + val x = "Hi" + x + }; def m8(x: String): String = + s"""Bye $x ${ + x + } +do not indent in a multiline string""" + def m9 = + val foo = "" + val x = Seq( + s"${foo}", + "" + ) + +// do not remove braces after closing brace + def m10(x: Int)(y: String) = y * x + m10 { 5 } { + "foo" + } + + // preserve indent of chained calls + def m11(xs: Seq[String]) = + xs + .filter: + _ => true + xs + .map { x => + val y = + if (x == "") "empty" + else x.size.toString + val z = x + y + z + } + .map { x => xs }.flatMap { xs => xs.map { x => + val y = + if (x == "") "empty" + else x.size.toString + val z = x + y + z + }} + .map: + x => val y = + if (x == "") "empty" + else x.size.toString + val z = x + y + z + .map: + x => + val y = + if (x == "") "empty" + else x.size.toString + val z = x + y + z + + // do not remove braces inside (...) or [...] + // remove braces after => + def m12(xs: List[Int]) = + println( + xs.size match { + case 1 => + xs match + case 1 :: Nil => "1" + case _ => s"${xs.head} :: Nil" + case _ => { + "xs" + } + } + ) + println( + if (xs.size > 0) { + "foo" + } else { + "bar" + } + ) + xs.map( + x => { + x + } + ).map: + x => { + x + } + import reflect.Selectable.reflectiveSelectable + def m13(xs: List[ + Any { + def foo: String + } + ]) = + xs.map(x => x.foo) + + // preserve indentation style before 'case' + // but fix indentation inside 'case' + def m14(o: Option[String]) = + o match + case Some(x) => x + case None => "" + + o match + case Some(x) => x + case None => "" + + o match + case None => + "" + case Some(x) => + x + def m15(xs: List[Int]): String = + xs match + case _ :: tail => { + if tail.size == 0 then + println("log") + } + "foo" + case Nil => + "bar" + + // add backticks around operator + object `*:`: + def foo = ??? + def m16 = + val x = 5 * { + 2 + } == 10 `||`: + false + x `&&`: + true + + // leading infix operator + def m17 = + true + && { + false + } + + // ident ending with '_' + def m_(x: String) = ??? + m_ : + "foo" + + // do not remove braces in sequence of blocks + def m18(using ctx: String) = println(ctx) + { + given String = "foo" + m18 + } + { + given String = "bar" + m18 + } + def m19(x: String) = + { + given String = "foo" + m18 + } + { + given String = "bar" + m18 + } + + // back-quote end indent before match + def m20 = + val end = "Foo" + `end` match + case "Foo" => + case _ => + end take 3 + +// indent template after self type +class C2 { self => + val x = "" +} +trait C3: + self => + val x = "" +case class C4(): + self => + val y = "" diff --git a/tests/rewrites/indent-rewrite.scala b/tests/rewrites/indent-rewrite.scala new file mode 100644 index 000000000000..a7bdf84409b9 --- /dev/null +++ b/tests/rewrites/indent-rewrite.scala @@ -0,0 +1,274 @@ +// A collection of patterns found when rewriting the community build to indent + +trait C1 { + + class CC1 +// do not remove braces if empty region +class CC2 { + +} +// do not remove braces if open brace is not followed by new line +def m1(x: Int) = +{ x +.toString + } +// add indent to pass an argument (fewer braces) +def m2: String = { +m1 { +5 +} +} +// indent inner method + def m3: Int = { +def seq = { +Seq( +"1", +"2" +) +} +seq +(1) +.toInt +} +// indent refinement +def m4: Any { +def foo: String +} += + new { + def foo: String = + """ +Hello, World! +""" +} +// indent end marker +end m4 + +// fix off-by-one indentation + val x = "" + + def m5(x: String): String = { + def inner: Boolean = { + true + } + x + } + + // unindent properly when needed + def m6(xs: Seq[String]): String = { + xs + .map { + x => x + } + .filter { + x => x.size > 0 + } + println("foo") + + def foo: String = + "" + foo + } + +// do not remove braces if closing braces not followed by new line +def m7: String = { +val x = "Hi" +x +}; def m8(x: String): String = { +s"""Bye $x ${ + x +} +do not indent in a multiline string""" +} + def m9 = { + val foo = "" + val x = Seq( + s"${foo}", + "" + ) + } + +// do not remove braces after closing brace +def m10(x: Int)(y: String) = y * x +m10 { 5 } { + "foo" +} + + // preserve indent of chained calls + def m11(xs: Seq[String]) = { + xs + .filter { + _ => true + } + xs + .map { x => + val y = + if (x == "") "empty" + else x.size.toString + val z = x + y + z + } + .map { x => xs }.flatMap { xs => xs.map { x => + val y = + if (x == "") "empty" + else x.size.toString + val z = x + y + z + }} + .map { + x => val y = + if (x == "") "empty" + else x.size.toString + val z = x + y + z + } + .map { + x => + val y = + if (x == "") "empty" + else x.size.toString + val z = x + y + z + } + } + + // do not remove braces inside (...) or [...] + // remove braces after => + def m12(xs: List[Int]) = { + println( + xs.size match { + case 1 => + xs match { + case 1 :: Nil => "1" + case _ => s"${xs.head} :: Nil" + } + case _ => { + "xs" + } + } + ) + println( + if (xs.size > 0) { + "foo" + } else { + "bar" + } + ) + xs.map( + x => { + x + } + ).map { + x => { + x + } + } + } + import reflect.Selectable.reflectiveSelectable + def m13(xs: List[ + Any { + def foo: String + } + ]) = + xs.map(x => x.foo) + + // preserve indentation style before 'case' + // but fix indentation inside 'case' + def m14(o: Option[String]) = { + o match + case Some(x) => x + case None => "" + + o match + case Some(x) => x + case None => "" + + o match { + case None => + "" + case Some(x) => + x + } + } + def m15(xs: List[Int]): String = { + xs match { + case _ :: tail => { + if tail.size == 0 then + println("log") + } + "foo" + case Nil => + "bar" + } + } + + // add backticks around operator + object *:{ + def foo = ??? + } + def m16 = + val x = 5 * { + 2 + } == 10 || { + false + } + x `&&` { + true + } + + // leading infix operator + def m17 = + true + && { + false + } + + // ident ending with '_' + def m_(x: String) = ??? + m_ { + "foo" + } + + // do not remove braces in sequence of blocks + def m18(using ctx: String) = println(ctx) + { + given String = "foo" + m18 + } + { + given String = "bar" + m18 + } + def m19(x: String) = { + { + given String = "foo" + m18 + } + { + given String = "bar" + m18 + } + } + + // back-quote end indent before match + def m20 = + val end = "Foo" + end match { + case "Foo" => + case _ => + } + end take 3 +} + +// indent template after self type +class C2 { self => +val x = "" +} +trait C3 { + self => +val x = "" +} +case class C4() { +self => + val y = "" +} From 351b5f51ff6b5729047aea25470e02f56c2b692f Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Tue, 13 Jun 2023 17:05:16 +0200 Subject: [PATCH 5/9] Fix end ident followed by MATCH --- compiler/src/dotty/tools/dotc/parsing/Scanners.scala | 3 +++ tests/rewrites/indent-rewrite.check | 1 + tests/rewrites/indent-rewrite.scala | 1 + 3 files changed, 5 insertions(+) diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index ea3960839f06..ef95f6b7be5e 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -766,6 +766,9 @@ object Scanners { if lookahead.token == EOF || source.offsetToLine(lookahead.offset) > endLine then return true + + if lookahead.token == LBRACE && rewriteToIndent then + patch(Span(offset, offset + 3), s"`end`") false /** Is there a blank line between the current token and the last one? diff --git a/tests/rewrites/indent-rewrite.check b/tests/rewrites/indent-rewrite.check index c18b123f2af7..592d1c9f4dd9 100644 --- a/tests/rewrites/indent-rewrite.check +++ b/tests/rewrites/indent-rewrite.check @@ -162,6 +162,7 @@ do not indent in a multiline string""" o match case Some(x) => x case None => "" + end match o match case None => diff --git a/tests/rewrites/indent-rewrite.scala b/tests/rewrites/indent-rewrite.scala index a7bdf84409b9..e00eaf535357 100644 --- a/tests/rewrites/indent-rewrite.scala +++ b/tests/rewrites/indent-rewrite.scala @@ -182,6 +182,7 @@ m10 { 5 } { o match case Some(x) => x case None => "" + end match o match { case None => From a0d28a3a68def88eafdcfc6116c18840f64774c9 Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Fri, 16 Jun 2023 15:50:53 +0200 Subject: [PATCH 6/9] Remove inStatSeq and other requested changes --- .../dotty/tools/dotc/parsing/Parsers.scala | 67 +++++++++---------- .../dotty/tools/dotc/rewrites/Rewrites.scala | 3 +- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index caa97285b97a..a2cbd34297b3 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -559,7 +559,7 @@ object Parsers { def inBraces[T](body: => T): T = enclosed(LBRACE, body) def inBrackets[T](body: => T): T = enclosed(LBRACKET, body) - def inBracesOrIndented[T](body: => T, inStatSeq: Boolean = false, rewriteWithColon: Boolean = false): T = + def inBracesOrIndented[T](body: => T, rewriteWithColon: Boolean = false): T = if in.token == INDENT then // braces are always optional after `=>` so none should be inserted val afterArrow = testChars(in.lastOffset - 3, " =>") @@ -569,11 +569,11 @@ object Parsers { else if rewriteToIndent then enclosed(INDENT, toIndentedRegion(body)) else enclosed(INDENT, body) else - if in.rewriteToIndent then bracesToIndented(body, inStatSeq, rewriteWithColon) + if in.rewriteToIndent then bracesToIndented(body, rewriteWithColon) else inBraces(body) - def inDefScopeBraces[T](body: => T, inStatSeq: Boolean = false, rewriteWithColon: Boolean = false): T = - inBracesOrIndented(body, inStatSeq, rewriteWithColon) + def inDefScopeBraces[T](body: => T, rewriteWithColon: Boolean = false): T = + inBracesOrIndented(body, rewriteWithColon) /** {`,` } */ def commaSeparated[T](part: () => T): List[T] = @@ -748,6 +748,9 @@ object Parsers { def blankLinesAround(start: Offset, end: Offset): (Offset, Offset) = (skipBlanks(start - 1, -1) + 1, skipBlanks(end, 1)) + private val bracesToIndentPredecessors = + colonEOLPredecessors | canStartIndentTokens | BitSet(IDENTIFIER) + /** Parse brace-enclosed `body` and rewrite it to be an indentation region instead, if possible. * If possible means: * 1. not inside (...), [...], case ... => @@ -756,16 +759,10 @@ object Parsers { * 4. there is at least one token between the braces * 5. the closing brace is also at the end of the line, or it is followed by one of * `then`, `else`, `do`, `catch`, `finally`, `yield`, or `match`. - * 6. the opening brace does not follow a closing brace + * 6. the opening brace follows an colonEOLPredecessors, a canStartIndentTokens or an identifier * 7. last token is not a leading operator - * 8. not a block in a sequence of statements - * 9. cannot rewrite if colon required after a NEWLINE, e.g. - * true || - * { - * false - * } - */ - def bracesToIndented[T](body: => T, inStatSeq: Boolean, rewriteWithColon: Boolean): T = + */ + def bracesToIndented[T](body: => T, rewriteWithColon: Boolean): T = import IndentRewriteState.* val prevSaved = prev.saveCopy val lastOffsetSaved = in.lastOffset @@ -773,14 +770,12 @@ object Parsers { val colonRequired = rewriteWithColon || underColonSyntax val (startOpening, endOpening) = elimRegion(in.offset) def isBracesOrIndented(r: Region): Boolean = r match - case r: Indented => true - case r: InBraces => true + case r: (Indented | InBraces) => true case _ => false - var canRewrite = isBracesOrIndented(in.currentRegion) && // test (1) - prevSaved.token != RBRACE && // test (6) - !(prevSaved.isOperator && prevSaved.isAfterLineEnd) && // test (7) - !inStatSeq && // test (8) - (!colonRequired || !in.isAfterLineEnd) // test (9) + var canRewrite = + isBracesOrIndented(in.currentRegion) // test (1) + && bracesToIndentPredecessors.contains(prevSaved.token) // test (6) + && !(prevSaved.isOperator && prevSaved.isAfterLineEnd) // test (7) val t = enclosed(LBRACE, { if in.isAfterLineEnd && in.token != RBRACE then // test (2)(4) toIndentedRegion: @@ -2180,7 +2175,7 @@ object Parsers { def subExpr() = subPart(expr) - def expr(location: Location, inStatSeq: Boolean = false): Tree = { + def expr(location: Location): Tree = { val start = in.offset in.token match case IMPLICIT => @@ -2208,7 +2203,7 @@ object Parsers { else new WildcardFunction(placeholderParams.reverse, t) finally placeholderParams = saved - val t = expr1(location, inStatSeq) + val t = expr1(location) if in.isArrow then placeholderParams = Nil // don't interpret `_' to the left of `=>` as placeholder wrapPlaceholders(closureRest(start, location, convertToParams(t))) @@ -2220,7 +2215,7 @@ object Parsers { wrapPlaceholders(t) } - def expr1(location: Location = Location.ElseWhere, inStatSeq: Boolean = false): Tree = in.token match + def expr1(location: Location = Location.ElseWhere): Tree = in.token match case IF => ifExpr(in.offset, If) case WHILE => @@ -2318,7 +2313,7 @@ object Parsers { case t => syntaxError(em"`inline` must be followed by an `if` or a `match`", start) t - else expr1Rest(postfixExpr(location, inStatSeq), location) + else expr1Rest(postfixExpr(location), location) end expr1 def expr1Rest(t: Tree, location: Location): Tree = @@ -2476,8 +2471,8 @@ object Parsers { * | InfixExpr id ColonArgument * | InfixExpr MatchClause */ - def postfixExpr(location: Location = Location.ElseWhere, inStatSeq: Boolean = false): Tree = - val t = postfixExprRest(prefixExpr(location, inStatSeq), location) + def postfixExpr(location: Location = Location.ElseWhere): Tree = + val t = postfixExprRest(prefixExpr(location), location) if location.inArgs && followingIsVararg() then Typed(t, atSpan(skipToken()) { Ident(tpnme.WILDCARD_STAR) }) else @@ -2490,7 +2485,7 @@ object Parsers { /** PrefixExpr ::= [PrefixOperator'] SimpleExpr * PrefixOperator ::= ‘-’ | ‘+’ | ‘~’ | ‘!’ (if not backquoted) */ - def prefixExpr(location: Location, inStatSeq: Boolean = false): Tree = + val prefixExpr: Location => Tree = location => if in.token == IDENTIFIER && nme.raw.isUnary(in.name) && in.canStartExprTokens.contains(in.lookahead.token) then @@ -2500,7 +2495,7 @@ object Parsers { simpleExprRest(literal(start), location, canApply = true) else atSpan(start) { PrefixOp(op, simpleExpr(location)) } - else simpleExpr(location, inStatSeq) + else simpleExpr(location) /** SimpleExpr ::= ‘new’ ConstrApp {`with` ConstrApp} [TemplateBody] * | ‘new’ TemplateBody @@ -2526,7 +2521,7 @@ object Parsers { * Quoted ::= ‘'’ ‘{’ Block ‘}’ * | ‘'’ ‘[’ Type ‘]’ */ - def simpleExpr(location: Location, inStatSeq: Boolean = false): Tree = { + def simpleExpr(location: Location): Tree = { var canApply = true val t = in.token match { case XMLSTART => @@ -2547,7 +2542,7 @@ object Parsers { atSpan(in.offset) { makeTupleOrParens(inParens(exprsInParensOrBindings())) } case LBRACE | INDENT => canApply = false - blockExpr(inStatSeq) + blockExpr() case QUOTE => atSpan(skipToken()) { withinStaged(StageKind.Quoted | (if (location.inPattern) StageKind.QuotedPattern else 0)) { @@ -2711,12 +2706,12 @@ object Parsers { /** BlockExpr ::= <<< (CaseClauses | Block) >>> */ - def blockExpr(inStatSeq: Boolean = false): Tree = atSpan(in.offset) { + def blockExpr(): Tree = atSpan(in.offset) { val simplify = in.token == INDENT inDefScopeBraces({ if (in.token == CASE) Match(EmptyTree, caseClauses(() => caseClause())) else block(simplify) - }, inStatSeq = inStatSeq) + }) } /** Block ::= BlockStatSeq @@ -4177,7 +4172,7 @@ object Parsers { Template(constr, parents, derived, self, stats) def templateBody(parents: List[Tree], rewriteWithColon: Boolean = true): (ValDef, List[Tree]) = - val r = inDefScopeBraces(templateStatSeq(), rewriteWithColon = rewriteWithColon) + val r = inDefScopeBraces(templateStatSeq(), rewriteWithColon) if in.token == WITH && parents.isEmpty then syntaxError(EarlyDefinitionsNotSupported()) nextToken() @@ -4299,7 +4294,7 @@ object Parsers { else if (isDefIntro(modifierTokensOrCase)) stats +++= defOrDcl(in.offset, defAnnotsMods(modifierTokens)) else if (isExprIntro) - stats += expr1(inStatSeq = true) + stats += expr1() else empty = true statSepOrEnd(stats, noPrevStat = empty) @@ -4376,7 +4371,7 @@ object Parsers { if (in.token == IMPORT) stats ++= importClause() else if (isExprIntro) - stats += expr(Location.InBlock, inStatSeq = true) + stats += expr(Location.InBlock) else if in.token == IMPLICIT && !in.inModifierPosition() then stats += closure(in.offset, Location.InBlock, modifiers(BitSet(IMPLICIT))) else if isIdent(nme.extension) && followingIsExtension() then @@ -4445,7 +4440,7 @@ object Parsers { def skipBracesHook(): Option[Tree] = if (in.token == XMLSTART) Some(xmlLiteral()) else None - override def blockExpr(inStatSeq: Boolean): Tree = { + override def blockExpr(): Tree = { skipBraces() EmptyTree } diff --git a/compiler/src/dotty/tools/dotc/rewrites/Rewrites.scala b/compiler/src/dotty/tools/dotc/rewrites/Rewrites.scala index 45cc3c4ccfe0..bb9597dbc647 100644 --- a/compiler/src/dotty/tools/dotc/rewrites/Rewrites.scala +++ b/compiler/src/dotty/tools/dotc/rewrites/Rewrites.scala @@ -26,7 +26,8 @@ object Rewrites { pbuf += Patch(span, replacement) def patchOver(span: Span, replacement: String): Unit = - pbuf.indices.reverse.find(i => span.contains(pbuf(i).span)).foreach(pbuf.remove) + val prevPatchIdx = pbuf.lastIndexWhere(p => span.contains(p.span)) + if prevPatchIdx >= 0 then pbuf.remove(prevPatchIdx) pbuf += Patch(span, replacement) def apply(cs: Array[Char]): Array[Char] = { From d30ab4d95442860ef4d780e5d655734a7912b1c3 Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Fri, 16 Jun 2023 17:06:15 +0200 Subject: [PATCH 7/9] Introduce ToIndentParser and ToIndentScanner Remove ScriptParsers --- .../src/dotty/tools/dotc/core/Comments.scala | 4 +- .../dotty/tools/dotc/inlines/Inlines.scala | 4 +- .../tools/dotc/parsing/ParserPhase.scala | 2 +- .../dotty/tools/dotc/parsing/Parsers.scala | 585 +++++++++--------- .../tools/dotc/parsing/ScriptParsers.scala | 147 ----- .../dotc/printing/SyntaxHighlighting.scala | 4 +- .../src/dotty/tools/dotc/typer/Namer.scala | 4 +- .../src/dotty/tools/repl/ParseResult.scala | 4 +- .../tools/dotc/ast/UntypedTreeMapTest.scala | 4 +- 9 files changed, 306 insertions(+), 452 deletions(-) delete mode 100644 compiler/src/dotty/tools/dotc/parsing/ScriptParsers.scala diff --git a/compiler/src/dotty/tools/dotc/core/Comments.scala b/compiler/src/dotty/tools/dotc/core/Comments.scala index 1b20b75ad8ac..f8e531e89888 100644 --- a/compiler/src/dotty/tools/dotc/core/Comments.scala +++ b/compiler/src/dotty/tools/dotc/core/Comments.scala @@ -10,7 +10,7 @@ import util.{SourceFile, ReadOnlyMap} import util.Spans._ import util.CommentParsing._ import util.Property.Key -import parsing.Parsers.Parser +import parsing.Parsers import reporting.ProperDefinitionNotFound object Comments { @@ -125,7 +125,7 @@ object Comments { object UseCase { def apply(code: String, codePos: Span)(using Context): UseCase = { val tree = { - val tree = new Parser(SourceFile.virtual("", code)).localDef(codePos.start) + val tree = Parsers.parser(SourceFile.virtual("", code)).localDef(codePos.start) tree match { case tree: untpd.DefDef => val newName = ctx.compilationUnit.freshNames.newName(tree.name, NameKinds.DocArtifactName) diff --git a/compiler/src/dotty/tools/dotc/inlines/Inlines.scala b/compiler/src/dotty/tools/dotc/inlines/Inlines.scala index 36dc8a642afc..e36a6baff168 100644 --- a/compiler/src/dotty/tools/dotc/inlines/Inlines.scala +++ b/compiler/src/dotty/tools/dotc/inlines/Inlines.scala @@ -12,7 +12,7 @@ import SymDenotations.SymDenotation import config.Printers.inlining import ErrorReporting.errorTree import dotty.tools.dotc.util.{SourceFile, SourcePosition, SrcPos} -import parsing.Parsers.Parser +import parsing.Parsers import transform.{PostTyper, Inlining, CrossVersionChecks} import staging.StagingLevel @@ -332,7 +332,7 @@ object Inlines: case ConstantType(Constant(code: String)) => val source2 = SourceFile.virtual("tasty-reflect", code) inContext(ctx.fresh.setNewTyperState().setTyper(new Typer(ctx.nestingLevel + 1)).setSource(source2)) { - val tree2 = new Parser(source2).block() + val tree2 = Parsers.parser(source2).block() if ctx.reporter.allErrors.nonEmpty then ctx.reporter.allErrors.map((ErrorKind.Parser, _)) else diff --git a/compiler/src/dotty/tools/dotc/parsing/ParserPhase.scala b/compiler/src/dotty/tools/dotc/parsing/ParserPhase.scala index a67bca34cae2..c2f130996025 100644 --- a/compiler/src/dotty/tools/dotc/parsing/ParserPhase.scala +++ b/compiler/src/dotty/tools/dotc/parsing/ParserPhase.scala @@ -27,7 +27,7 @@ class Parser extends Phase { unit.untpdTree = if (unit.isJava) new JavaParsers.JavaParser(unit.source).parse() else { - val p = new Parsers.Parser(unit.source) + val p = Parsers.parser(unit.source) // p.in.debugTokenStream = true val tree = p.parse() if (p.firstXmlPos.exists && !firstXmlPos.exists) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index a2cbd34297b3..70969491e758 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -23,7 +23,6 @@ import StdNames._ import util.Spans._ import Constants._ import Symbols.NoSymbol -import ScriptParsers._ import Decorators._ import util.Chars import scala.annotation.tailrec @@ -74,7 +73,7 @@ object Parsers { * if not, the AST will be supplemented. */ def parser(source: SourceFile)(using Context): Parser = - if source.isSelfContained then new ScriptParser(source) + if ctx.settings.rewrite.value.isDefined && ctx.settings.indent.value then new ToIndentParser(source) else new Parser(source) private val InCase: Region => Region = Scanners.InCase(_) @@ -169,13 +168,12 @@ object Parsers { } } - class Parser(source: SourceFile, allowRewrite: Boolean = true)(using Context) extends ParserCommon(source) { + class Parser private[parsing] (source: SourceFile, allowRewrite: Boolean = true)(using Context) extends ParserCommon(source) { val in: Scanner = new Scanner(source, profile = Profile.current, allowRewrite = allowRewrite) // in.debugTokenStream = true // uncomment to see the token stream of the standard scanner, but not syntax highlighting /** This is the general parse entry point. - * Overridden by ScriptParser */ def parse(): Tree = { val t = compilationUnit() @@ -297,7 +295,7 @@ object Parsers { val offset = in.offset if in.token != token then syntaxErrorOrIncomplete(ExpectedTokenButFound(token, in.token)) - if in.token == token then nextToken() + if in.token == token then in.nextToken() offset def accept(name: Name): Int = { @@ -305,20 +303,20 @@ object Parsers { if !isIdent(name) then syntaxErrorOrIncomplete(em"`$name` expected") if isIdent(name) then - nextToken() + in.nextToken() offset } def acceptColon(): Int = val offset = in.offset - if in.isColon then { nextToken(); offset } + if in.isColon then { in.nextToken(); offset } else accept(COLONop) /** semi = nl {nl} | `;' * nl = `\n' // where allowed */ def acceptStatSep(): Unit = - if in.isNewLine then nextToken() else accept(SEMI) + if in.isNewLine then in.nextToken() else accept(SEMI) /** Parse statement separators and end markers. Ensure that there is at least * one statement separator unless the next token terminates a statement´sequence. @@ -333,7 +331,7 @@ object Parsers { def statSepOrEnd[T <: Tree](stats: ListBuffer[T], noPrevStat: Boolean = false, what: String = "statement", altEnd: Token = EOF): Boolean = def recur(sepSeen: Boolean, endSeen: Boolean): Boolean = if isStatSep then - nextToken() + in.nextToken() recur(true, endSeen) else if in.token == END then if endSeen then syntaxError(em"duplicate end marker") @@ -352,7 +350,7 @@ object Parsers { if mustStartStatTokens.contains(found) then false // it's a statement that might be legal in an outer context else - nextToken() // needed to ensure progress; otherwise we might cycle forever + in.nextToken() // needed to ensure progress; otherwise we might cycle forever skip() true @@ -473,7 +471,6 @@ object Parsers { } case _ => - /** Convert (qual)ident to type identifier */ def convertToTypeId(tree: Tree): Tree = tree match { @@ -559,18 +556,14 @@ object Parsers { def inBraces[T](body: => T): T = enclosed(LBRACE, body) def inBrackets[T](body: => T): T = enclosed(LBRACKET, body) + def isAfterArrow = testChars(in.lastOffset - 3, " =>") + def inBracesOrIndented[T](body: => T, rewriteWithColon: Boolean = false): T = if in.token == INDENT then // braces are always optional after `=>` so none should be inserted - val afterArrow = testChars(in.lastOffset - 3, " =>") - val rewriteToBraces = in.rewriteNoIndent && !afterArrow - val rewriteToIndent = in.rewriteToIndent && !afterArrow - if rewriteToBraces then indentedToBraces(body) - else if rewriteToIndent then enclosed(INDENT, toIndentedRegion(body)) + if in.rewriteNoIndent && !isAfterArrow then indentedToBraces(body) else enclosed(INDENT, body) - else - if in.rewriteToIndent then bracesToIndented(body, rewriteWithColon) - else inBraces(body) + else inBraces(body) def inDefScopeBraces[T](body: => T, rewriteWithColon: Boolean = false): T = inBracesOrIndented(body, rewriteWithColon) @@ -589,7 +582,7 @@ object Parsers { if in.token == COMMA then val ts = new ListBuffer[T] += leading while in.token == COMMA do - nextToken() + in.nextToken() ts += part() ts.toList else leading :: Nil @@ -641,23 +634,6 @@ object Parsers { /* -------- REWRITES ----------------------------------------------------------- */ - object IndentRewriteState: - assert(in.rewriteToIndent) - - /** A copy of the previous token */ - var prev: TokenData = Scanners.newTokenData - - /** The last offset where a colon at the end of line would be required if a subsequent { ... } - * block would be converted to an indentation region. */ - var possibleColonOffset: Int = -1 - - /** When rewriting to indent, the minimum indent width to rewrite to */ - var minimumIndent: IndentWidth = IndentWidth.Zero - - /** When rewritting to indent, the maximum indent width to rewrite to - * to ensure an indent region is properly closed by outdentation */ - var maximumIndent: Option[IndentWidth] = None - def testChar(idx: Int, p: Char => Boolean): Boolean = val txt = source.content idx > -1 && idx < txt.length && p(txt(idx)) @@ -748,117 +724,9 @@ object Parsers { def blankLinesAround(start: Offset, end: Offset): (Offset, Offset) = (skipBlanks(start - 1, -1) + 1, skipBlanks(end, 1)) - private val bracesToIndentPredecessors = - colonEOLPredecessors | canStartIndentTokens | BitSet(IDENTIFIER) - - /** Parse brace-enclosed `body` and rewrite it to be an indentation region instead, if possible. - * If possible means: - * 1. not inside (...), [...], case ... => - * 2. opening brace `{` is at end of line - * 3. closing brace `}` is at start of line - * 4. there is at least one token between the braces - * 5. the closing brace is also at the end of the line, or it is followed by one of - * `then`, `else`, `do`, `catch`, `finally`, `yield`, or `match`. - * 6. the opening brace follows an colonEOLPredecessors, a canStartIndentTokens or an identifier - * 7. last token is not a leading operator - */ - def bracesToIndented[T](body: => T, rewriteWithColon: Boolean): T = - import IndentRewriteState.* - val prevSaved = prev.saveCopy - val lastOffsetSaved = in.lastOffset - val underColonSyntax = possibleColonOffset == in.lastOffset - val colonRequired = rewriteWithColon || underColonSyntax - val (startOpening, endOpening) = elimRegion(in.offset) - def isBracesOrIndented(r: Region): Boolean = r match - case r: (Indented | InBraces) => true - case _ => false - var canRewrite = - isBracesOrIndented(in.currentRegion) // test (1) - && bracesToIndentPredecessors.contains(prevSaved.token) // test (6) - && !(prevSaved.isOperator && prevSaved.isAfterLineEnd) // test (7) - val t = enclosed(LBRACE, { - if in.isAfterLineEnd && in.token != RBRACE then // test (2)(4) - toIndentedRegion: - try body - finally canRewrite &= in.isAfterLineEnd // test (3) - else - canRewrite = false - body - }) - canRewrite &= (in.isAfterLineEnd || in.token == EOF || statCtdTokens.contains(in.token)) // test (5) - if canRewrite && (!underColonSyntax || Feature.fewerBracesEnabled) then - val (startClosing, endClosing) = elimRegion(in.lastOffset - 1) - // patch over the added indentation to remove braces - patchOver(source, Span(startOpening, endOpening), "") - patchOver(source, Span(startClosing, endClosing), "") - if colonRequired then - if prevSaved.token == IDENTIFIER && prevSaved.isOperator then - patch(Span(prevSaved.offset, lastOffsetSaved), s"`${prevSaved.name}`:") - else if prevSaved.token == IDENTIFIER && prevSaved.name.last == '_' then - patch(Span(lastOffsetSaved), " :") - else patch(Span(lastOffsetSaved), ":") - else - // no need to force outdentation after `}` - maximumIndent = None - t - end bracesToIndented - /** When rewriting to indent, make sure there is an indent after a `=>\n` */ def indentedRegionAfterArrow[T](body: => T, inCaseDef: Boolean = false): T = - if in.rewriteToIndent && (inCaseDef || in.isAfterLineEnd) then - // assert(in.last.isArrow || in.last.token == SELFARROW) - toIndentedRegion(body) - else body - - /** compute required indentation to indent region properly */ - def toIndentedRegion[T](body: => T): T = - import IndentRewriteState.* - val enclosingIndent = minimumIndent - minimumIndent = - if enclosingIndent < in.currentRegion.indentWidth then - in.currentRegion.indentWidth - else if - in.token == CASE && ( - in.currentRegion.enclosing == null || - in.currentRegion.indentWidth == in.currentRegion.enclosing.indentWidth - ) - then enclosingIndent - else enclosingIndent.increment - try body - finally - maximumIndent = Some(minimumIndent) - minimumIndent = enclosingIndent - - /** when rewriting to indent, check that indentaton is correct or patch */ - def patchIndent(): Unit = - if in.isAfterLineEnd && !in.isNewLine && in.token != OUTDENT && in.token != INDENT then - import IndentRewriteState.* - val currentIndent = in.indentWidth(in.offset) - val indentEndOffset = in.lineOffset + currentIndent.size - def isDotOrClosing = (closingParens + DOT).contains(in.token) - val needsOutdent = maximumIndent.exists: max => - currentIndent >= max || (!isDotOrClosing && currentIndent > minimumIndent) - val offByOne = - currentIndent != minimumIndent && currentIndent.isClose(minimumIndent) - if needsOutdent || !(currentIndent >= minimumIndent) || offByOne then - patch(Span(in.lineOffset, indentEndOffset), minimumIndent.toPrefix) - // no need to outdent anymore - if in.token != RBRACE then - maximumIndent = None - - def nextToken(): Unit = - if in.rewriteToIndent then - IndentRewriteState.prev = in.saveCopy - patchIndent() - in.nextToken() - - def skipToken(): Offset = - if in.rewriteToIndent then patchIndent() - in.skipToken() - - def skipToken[T](res: T): T = - if in.rewriteToIndent then patchIndent() - in.skipToken(res) + body /** Drop (...) or { ... }, replacing the closing element with `endStr` */ def dropParensOrBraces(start: Offset, endStr: String): Unit = { @@ -1097,7 +965,7 @@ object Parsers { colonAtEOLOpt() newLineOptWhenFollowing(canStartOperand) if isColonLambda then - nextToken() + in.nextToken() recur(expr(Location.InColonArg)) else if maybePostfix && !canStartOperand(in.token) then val topInfo = opStack.head @@ -1131,7 +999,7 @@ object Parsers { report.error( em"""Illegal backquoted identifier: `` and `` are forbidden""", in.sourcePos()) - nextToken() + in.nextToken() name else syntaxErrorOrIncomplete(ExpectedTokenButFound(IDENTIFIER, in.token)) @@ -1169,7 +1037,7 @@ object Parsers { /** DotSelectors ::= { `.' id } */ def dotSelectors(t: Tree): Tree = - if (in.token == DOT) { nextToken(); dotSelectors(selector(t)) } + if (in.token == DOT) { in.nextToken(); dotSelectors(selector(t)) } else t private val id: Tree => Tree = x => x @@ -1182,11 +1050,11 @@ object Parsers { val start = in.offset def handleThis(qual: Ident) = - nextToken() + in.nextToken() atSpan(start) { This(qual) } def handleSuper(qual: Ident) = - nextToken() + in.nextToken() val mix = mixinQualifierOpt() val t = atSpan(start) { Super(This(qual), mix) } accept(DOT) @@ -1200,10 +1068,10 @@ object Parsers { def qual = cpy.Ident(t)(t.name.toTypeName) in.lookahead.token match case THIS => - nextToken() + in.nextToken() handleThis(qual) case SUPER => - nextToken() + in.nextToken() handleSuper(qual) case _ => t else t @@ -1238,7 +1106,7 @@ object Parsers { def simpleLiteral(): Tree = if isIdent(nme.raw.MINUS) then val start = in.offset - nextToken() + in.nextToken() literal(negOffset = start, inTypeOrSingleton = true) else literal(inTypeOrSingleton = true) @@ -1294,7 +1162,7 @@ object Parsers { syntaxErrorOrIncomplete(IllegalLiteral()) atSpan(negOffset) { Literal(Constant(null)) } } - nextToken() + in.nextToken() t } else atSpan(negOffset) { @@ -1308,7 +1176,7 @@ object Parsers { case _ => Ident(in.name) } } - nextToken() + in.nextToken() Quote(t, Nil) } else @@ -1324,11 +1192,11 @@ object Parsers { if migrateTo3 then patch(source, Span(in.offset, in.offset + 1), "Symbol(\"") patch(source, Span(in.charOffset - 1), "\")") - atSpan(skipToken()) { SymbolLit(in.strVal) } + atSpan(in.skipToken()) { SymbolLit(in.strVal) } else if (in.token == INTERPOLATIONID) interpolatedString(inPattern) else { val t = literalOf(in.token) - nextToken() + in.nextToken() t } } @@ -1341,7 +1209,7 @@ object Parsers { in.charOffset + 1 < in.buf.length && in.buf(in.charOffset) == '"' && in.buf(in.charOffset + 1) == '"' - nextToken() + in.nextToken() def nextSegment(literalOffset: Offset) = segmentBuf += Thicket( literal(literalOffset, inPattern = inPattern, inStringInterpolation = true), @@ -1349,11 +1217,11 @@ object Parsers { if (in.token == IDENTIFIER) termIdent() else if (in.token == USCORE && inPattern) { - nextToken() + in.nextToken() Ident(nme.WILDCARD) } else if (in.token == THIS) { - nextToken() + in.nextToken() This(EmptyTypeIdent) } else if (in.token == LBRACE) @@ -1378,21 +1246,21 @@ object Parsers { /* ------------- NEW LINES ------------------------------------------------- */ def newLineOpt(): Unit = - if in.token == NEWLINE then nextToken() + if in.token == NEWLINE then in.nextToken() def newLinesOpt(): Unit = - if in.isNewLine then nextToken() + if in.isNewLine then in.nextToken() def newLineOptWhenFollowedBy(token: Int): Unit = // note: next is defined here because current == NEWLINE - if in.token == NEWLINE && in.next.token == token then nextToken() + if in.token == NEWLINE && in.next.token == token then in.nextToken() def newLinesOptWhenFollowedBy(token: Int): Unit = - if in.isNewLine && in.next.token == token then nextToken() + if in.isNewLine && in.next.token == token then in.nextToken() def newLinesOptWhenFollowedBy(name: Name): Unit = if in.isNewLine && in.next.token == IDENTIFIER && in.next.name == name then - nextToken() + in.nextToken() def newLineOptWhenFollowing(p: Int => Boolean): Unit = // note: next is defined here because current == NEWLINE @@ -1403,17 +1271,15 @@ object Parsers { syntaxErrorOrIncomplete(em"indented definitions expected, ${in} found") def colonAtEOLOpt(): Unit = - if in.rewriteToIndent then - IndentRewriteState.possibleColonOffset = in.lastOffset in.observeColonEOL(inTemplate = false) if in.token == COLONeol then - nextToken() + in.nextToken() acceptIndent() def argumentStart(): Unit = colonAtEOLOpt() if migrateTo3 && in.token == NEWLINE && in.next.token == LBRACE then - nextToken() + in.nextToken() if in.indentWidth(in.offset) == in.currentRegion.indentWidth then report.errorOrMigrationWarning( em"""This opening brace will start a new statement in Scala 3. @@ -1427,7 +1293,7 @@ object Parsers { if in.token == COLONeol then if in.lookahead.token == END then in.token = NEWLINE else - nextToken() + in.nextToken() if in.token != LBRACE then acceptIndent() else newLineOptWhenFollowedBy(LBRACE) @@ -1467,11 +1333,11 @@ object Parsers { didMatch if in.token == END then - val start = skipToken() + val start = in.skipToken() if stats.isEmpty || !matchesAndSetEnd(stats.last) then syntaxError(em"misaligned end marker", Span(start, in.lastCharOffset)) in.token = IDENTIFIER // Leaving it as the original token can confuse newline insertion - nextToken() + in.nextToken() end checkEndMarker /* ------------- TYPES ------------------------------------------------------ */ @@ -1563,10 +1429,10 @@ object Parsers { imods |= Impure if token == CTXARROW then - nextToken() + in.nextToken() imods |= Given else if token == ARROW || token == TLARROW then - nextToken() + in.nextToken() else accept(ARROW) @@ -1591,16 +1457,16 @@ object Parsers { val t = if (in.token == LPAREN) { - nextToken() + in.nextToken() if (in.token == RPAREN) { - nextToken() + in.nextToken() functionRest(Nil) } else { val paramStart = in.offset def addErased() = erasedArgs.addOne(isErasedKw) - if isErasedKw then { skipToken(); } + if isErasedKw then { in.skipToken(); } addErased() val ts = in.currentRegion.withCommasExpected { funArgType() match @@ -1646,9 +1512,9 @@ object Parsers { val start = in.offset val tparams = typeParamClause(ParamOwner.TypeParam) if (in.token == TLARROW) - atSpan(start, skipToken())(LambdaTypeTree(tparams, toplevelTyp())) + atSpan(start, in.skipToken())(LambdaTypeTree(tparams, toplevelTyp())) else if (in.token == ARROW || isPureArrow(nme.PUREARROW)) { - val arrowOffset = skipToken() + val arrowOffset = in.skipToken() val body = toplevelTyp() atSpan(start, arrowOffset) { getFunction(body) match { @@ -1768,7 +1634,7 @@ object Parsers { }) else if Feature.ccEnabled && in.isIdent(nme.UPARROW) && isCaptureUpArrow then val upArrowStart = in.offset - nextToken() + in.nextToken() def cs = if in.token == LBRACE then captureSet() else atSpan(upArrowStart)(captureRoot) :: Nil @@ -1784,7 +1650,7 @@ object Parsers { def withTypeRest(t: Tree): Tree = if in.token == WITH then val withOffset = in.offset - nextToken() + in.nextToken() if in.token == LBRACE || in.token == INDENT then t else @@ -1820,12 +1686,12 @@ object Parsers { val inPattern = (staged & StageKind.QuotedPattern) != 0 val expr = if (in.name.length == 1) { - nextToken() + in.nextToken() withinStaged(StageKind.Spliced)(if (inPattern) inBraces(pattern()) else stagedBlock()) } else atSpan(in.offset + 1) { val id = Ident(in.name.drop(1)) - nextToken() + in.nextToken() id } if isType then @@ -1853,23 +1719,23 @@ object Parsers { SingletonTypeTree(simpleLiteral()) else if in.token == USCORE then if ctx.settings.YkindProjector.value == "underscores" then - val start = skipToken() + val start = in.skipToken() Ident(tpnme.USCOREkw).withSpan(Span(start, in.lastOffset, start)) else if sourceVersion.isAtLeast(future) then deprecationWarning(em"`_` is deprecated for wildcard arguments of types: use `?` instead") patch(source, Span(in.offset, in.offset + 1), "?") - val start = skipToken() + val start = in.skipToken() typeBounds().withSpan(Span(start, in.lastOffset, start)) // Allow symbols -_ and +_ through for compatibility with code written using kind-projector in Scala 3 underscore mode. // While these signify variant type parameters in Scala 2 + kind-projector, we ignore their variance markers since variance is inferred. else if (isIdent(nme.MINUS) || isIdent(nme.PLUS)) && in.lookahead.token == USCORE && ctx.settings.YkindProjector.value == "underscores" then val identName = in.name.toTypeName ++ nme.USCOREkw - val start = skipToken() - nextToken() + val start = in.skipToken() + in.nextToken() Ident(identName).withSpan(Span(start, in.lastOffset, start)) else if isIdent(nme.?) then - val start = skipToken() + val start = in.skipToken() typeBounds().withSpan(Span(start, in.lastOffset, start)) else def singletonArgs(t: Tree): Tree = @@ -1899,9 +1765,9 @@ object Parsers { else def singletonCompletion(t: Tree): Tree = if in.token == DOT then - nextToken() + in.nextToken() if in.token == TYPE then - nextToken() + in.nextToken() atSpan(startOffset(t)) { SingletonTypeTree(t) } else singletonCompletion(selector(t)) @@ -1991,7 +1857,7 @@ object Parsers { in.currentRegion.withCommasExpected { argType() match { case Ident(name) if in.token == EQUALS => - nextToken() + in.nextToken() commaSeparatedRest(NamedArg(name, argType()), () => namedTypeArg()) case firstArg => commaSeparatedRest(firstArg, () => argType()) @@ -2003,7 +1869,7 @@ object Parsers { def paramTypeOf(core: () => Tree): Tree = if in.token == ARROW || isPureArrow(nme.PUREARROW) then val isImpure = in.token == ARROW - atSpan(skipToken()): + atSpan(in.skipToken()): val tp = if isImpure then core() else capturesAndResult(core) if isImpure && Feature.pureFunsEnabled then ImpureByNameTypeTree(tp) else ByNameTypeTree(tp) @@ -2014,7 +1880,7 @@ object Parsers { if in.isIdent(nme.into) && in.featureEnabled(Feature.into) && canStartTypeTokens.contains(in.lookahead.token) - then atSpan(skipToken()) { Into(tp()) } + then atSpan(in.skipToken()) { Into(tp()) } else tp() /** FunArgType ::= Type @@ -2034,7 +1900,7 @@ object Parsers { def paramValueType(): Tree = { val t = maybeInto(toplevelTyp) if (isIdent(nme.raw.STAR)) { - nextToken() + in.nextToken() atSpan(startOffset(t)) { PostfixOp(t, Ident(tpnme.raw.STAR)) } } else t @@ -2059,7 +1925,7 @@ object Parsers { atSpan(in.offset) { TypeBoundsTree(bound(SUPERTYPE), bound(SUBTYPE)) } private def bound(tok: Int): Tree = - if (in.token == tok) { nextToken(); toplevelTyp() } + if (in.token == tok) { in.nextToken(); toplevelTyp() } else EmptyTree /** TypeParamBounds ::= TypeBounds {`<%' Type} {`:' Type} @@ -2073,21 +1939,21 @@ object Parsers { def contextBounds(pname: TypeName): List[Tree] = if in.isColon then - atSpan(skipToken()) { + atSpan(in.skipToken()) { AppliedTypeTree(toplevelTyp(), Ident(pname)) } :: contextBounds(pname) else if in.token == VIEWBOUND then report.errorOrMigrationWarning( em"view bounds `<%' are no longer supported, use a context bound `:' instead", in.sourcePos(), from = `3.0`) - atSpan(skipToken()) { + atSpan(in.skipToken()) { Function(Ident(pname) :: Nil, toplevelTyp()) } :: contextBounds(pname) else Nil def typedOpt(): Tree = - if in.isColon then { nextToken(); toplevelTyp() } + if in.isColon then { in.nextToken(); toplevelTyp() } else TypeTree().withSpan(Span(in.lastOffset)) def typeDependingOn(location: Location): Tree = @@ -2219,7 +2085,7 @@ object Parsers { case IF => ifExpr(in.offset, If) case WHILE => - atSpan(skipToken()) { + atSpan(in.skipToken()) { val cond = condExpr(DO) newLinesOpt() val body = subExpr() @@ -2230,10 +2096,10 @@ object Parsers { em"""`do while ` is no longer supported, |use `while ; do ()` instead.${rewriteNotice()}""", in.sourcePos(), from = `3.0`) - val start = skipToken() + val start = in.skipToken() atSpan(start) { val body = expr() - if (isStatSep) nextToken() + if (isStatSep) in.nextToken() val whileStart = in.offset accept(WHILE) val cond = expr() @@ -2251,12 +2117,12 @@ object Parsers { } case TRY => val tryOffset = in.offset - atSpan(skipToken()) { + atSpan(in.skipToken()) { val body = expr() val (handler, handlerStart) = if in.token == CATCH then val span = in.offset - nextToken() + in.nextToken() (if in.token == CASE then Match(EmptyTree, caseClause(exprOnly = true) :: Nil) else subExpr(), span) @@ -2274,7 +2140,7 @@ object Parsers { val finalizer = if (in.token == FINALLY) { - nextToken(); + in.nextToken(); val expr = subExpr() if expr.span.exists then expr else Literal(Constant(())) // finally without an expression @@ -2290,9 +2156,9 @@ object Parsers { ParsedTry(body, handler, finalizer) } case THROW => - atSpan(skipToken()) { Throw(expr()) } + atSpan(in.skipToken()) { Throw(expr()) } case RETURN => - atSpan(skipToken()) { + atSpan(in.skipToken()) { Return(if (isExprIntro) expr() else EmptyTree, EmptyTree) } case FOR => @@ -2302,7 +2168,7 @@ object Parsers { && !in.inModifierPosition() && in.canStartExprTokens.contains(in.lookahead.token) then - val start = skipToken() + val start = in.skipToken() in.token match case IF => ifExpr(start, InlineIf) @@ -2320,14 +2186,14 @@ object Parsers { if in.token == EQUALS then t match case Ident(_) | Select(_, _) | Apply(_, _) | PrefixOp(_, _) => - atSpan(startOffset(t), skipToken()) { + atSpan(startOffset(t), in.skipToken()) { val loc = if location.inArgs then location else Location.ElseWhere Assign(t, subPart(() => expr(loc))) } case _ => t else if in.isColon then - nextToken() + in.nextToken() ascription(t, location) else t @@ -2335,9 +2201,9 @@ object Parsers { def ascription(t: Tree, location: Location): Tree = atSpan(startOffset(t)) { in.token match { case USCORE if in.lookahead.isIdent(nme.raw.STAR) => - val uscoreStart = skipToken() + val uscoreStart = in.skipToken() val isVarargSplice = location.inArgs && followingIsVararg() - nextToken() + in.nextToken() if isVarargSplice then report.errorOrMigrationWarning( em"The syntax `x: _*` is no longer supported for vararg splices; use `x*` instead${rewriteNotice(`future-migration`)}", @@ -2370,11 +2236,11 @@ object Parsers { * `if' Expr `then' Expr [[semi] else Expr] */ def ifExpr(start: Offset, mkIf: (Tree, Tree, Tree) => If): If = - atSpan(start, skipToken()) { + atSpan(start, in.skipToken()) { val cond = condExpr(THEN) newLinesOpt() val thenp = subExpr() - val elsep = if (in.token == ELSE) { nextToken(); subExpr() } + val elsep = if (in.token == ELSE) { in.nextToken(); subExpr() } else EmptyTree mkIf(cond, thenp, elsep) } @@ -2382,7 +2248,7 @@ object Parsers { /** MatchClause ::= `match' `{' CaseClauses `}' */ def matchClause(t: Tree): Match = - atSpan(startOffset(t), skipToken()) { + atSpan(startOffset(t), in.skipToken()) { Match(t, inBracesOrIndented(caseClauses(() => caseClause()))) } @@ -2400,7 +2266,7 @@ object Parsers { */ def funParams(mods: Modifiers, location: Location): List[Tree] = if in.token == LPAREN then - nextToken() + in.nextToken() if in.token == RPAREN then Nil else @@ -2417,7 +2283,7 @@ object Parsers { em"This syntax is no longer supported; parameter needs to be enclosed in (...)${rewriteNotice(`future-migration`)}", source.atSpan(Span(start, in.lastOffset)), from = future) - nextToken() + in.nextToken() val t = infixType() if (sourceVersion == `future-migration`) { patch(source, Span(start), "(") @@ -2439,7 +2305,7 @@ object Parsers { def bindingName(): TermName = if (in.token == USCORE) { - nextToken() + in.nextToken() WildcardParamName.fresh() } else ident() @@ -2455,7 +2321,7 @@ object Parsers { if in.token == CTXARROW then if params.isEmpty then syntaxError(em"context function literals require at least one formal parameter", Span(start, in.lastOffset)) - nextToken() + in.nextToken() else accept(ARROW) val body = indentedRegionAfterArrow: @@ -2474,7 +2340,7 @@ object Parsers { def postfixExpr(location: Location = Location.ElseWhere): Tree = val t = postfixExprRest(prefixExpr(location), location) if location.inArgs && followingIsVararg() then - Typed(t, atSpan(skipToken()) { Ident(tpnme.WILDCARD_STAR) }) + Typed(t, atSpan(in.skipToken()) { Ident(tpnme.WILDCARD_STAR) }) else t @@ -2532,7 +2398,7 @@ object Parsers { case BACKQUOTED_IDENT | THIS | SUPER => simpleRef() case USCORE => - val start = skipToken() + val start = in.skipToken() val pname = WildcardParamName.fresh() val param = ValDef(pname, TypeTree(), EmptyTree).withFlags(SyntheticTermParam) .withSpan(Span(start)) @@ -2544,7 +2410,7 @@ object Parsers { canApply = false blockExpr() case QUOTE => - atSpan(skipToken()) { + atSpan(in.skipToken()) { withinStaged(StageKind.Quoted | (if (location.inPattern) StageKind.QuotedPattern else 0)) { val body = if (in.token == LBRACKET) inBrackets(typ()) @@ -2556,14 +2422,14 @@ object Parsers { canApply = false newExpr() case MACRO => - val start = skipToken() + val start = in.skipToken() MacroTree(simpleExpr(Location.ElseWhere)) case _ => if isLiteral then literal() else if in.isColon then syntaxError(IllegalStartSimpleExpr(tokenString(in.token))) - nextToken() + in.nextToken() simpleExpr(location) else val start = in.lastOffset @@ -2577,7 +2443,7 @@ object Parsers { if (canApply) argumentStart() in.token match case DOT => - nextToken() + in.nextToken() simpleExprRest(selectorOrMatch(t), location, canApply = true) case LBRACKET => val tapp = atSpan(startOffset(t), in.offset) { TypeApply(t, typeArgs(namedOK = true, wildOK = false)) } @@ -2586,7 +2452,7 @@ object Parsers { val app = atSpan(startOffset(t), in.offset) { mkApply(t, argumentExprs()) } simpleExprRest(app, location, canApply = true) case USCORE => - atSpan(startOffset(t), skipToken()) { PostfixOp(t, Ident(nme.WILDCARD)) } + atSpan(startOffset(t), in.skipToken()) { PostfixOp(t, Ident(nme.WILDCARD)) } case _ => if in.isColon && location == Location.InParens && followingIsLambdaParams() then t match @@ -2599,7 +2465,7 @@ object Parsers { } case _ => t else if isColonLambda then - val app = atSpan(startOffset(t), skipToken()) { + val app = atSpan(startOffset(t), in.skipToken()) { Apply(t, expr(Location.InColonArg) :: Nil) } simpleExprRest(app, location, canApply = true) @@ -2610,7 +2476,7 @@ object Parsers { * | ‘new’ TemplateBody */ def newExpr(): Tree = - val start = skipToken() + val start = in.skipToken() def reposition(t: Tree) = t.withSpan(Span(start, in.lastOffset)) possibleTemplateStart() val parents = @@ -2648,7 +2514,7 @@ object Parsers { if in.token == RPAREN then (Nil, false) else if isIdent(nme.using) then - nextToken() + in.nextToken() (commaSeparated(argumentExpr), true) else (commaSeparated(argumentExpr), false) @@ -2732,7 +2598,7 @@ object Parsers { /** Guard ::= if PostfixExpr */ def guard(): Tree = - if (in.token == IF) { nextToken(); postfixExpr(Location.InGuard) } + if (in.token == IF) { in.nextToken(); postfixExpr(Location.InGuard) } else EmptyTree /** Enumerators ::= Generator {semi Enumerator | Guard} @@ -2741,7 +2607,7 @@ object Parsers { def enumeratorsRest(): List[Tree] = if (isStatSep) { - nextToken() + in.nextToken() if (in.token == DO || in.token == YIELD || in.token == RBRACE) Nil else enumerator() :: enumeratorsRest() } @@ -2758,14 +2624,14 @@ object Parsers { else if (in.token == CASE) generator() else { val pat = pattern1() - if (in.token == EQUALS) atSpan(startOffset(pat), skipToken()) { GenAlias(pat, subExpr()) } + if (in.token == EQUALS) atSpan(startOffset(pat), in.skipToken()) { GenAlias(pat, subExpr()) } else generatorRest(pat, casePat = false) } /** Generator ::= [‘case’] Pattern `<-' Expr */ def generator(): Tree = { - val casePat = if (in.token == CASE) { nextToken(); true } else false + val casePat = if (in.token == CASE) { in.nextToken(); true } else false generatorRest(pattern1(), casePat) } @@ -2784,14 +2650,14 @@ object Parsers { * | ‘for’ Enumerators (‘do‘ | ‘yield’) Expr */ def forExpr(): Tree = - atSpan(skipToken()) { + atSpan(in.skipToken()) { var wrappedEnums = true val start = in.offset val forEnd = in.lastOffset val leading = in.token val enums = if (leading == LBRACE || leading == LPAREN && followingIsEnclosedGenerators()) { - nextToken() + in.nextToken() val res = if (leading == LBRACE || in.token == CASE) enumerators() @@ -2843,12 +2709,12 @@ object Parsers { } newLinesOpt() if (in.token == YIELD) { - nextToken() + in.nextToken() ForYield(enums, subExpr()) } else if (in.token == DO) { if (rewriteToOldSyntax()) dropTerminator() - nextToken() + in.nextToken() ForDo(enums, subExpr()) } else { @@ -2896,7 +2762,7 @@ object Parsers { accept(CASE) in.token match { case USCORE if in.lookahead.isArrow => - val start = skipToken() + val start = in.skipToken() Ident(tpnme.WILDCARD).withSpan(Span(start, in.lastOffset, start)) case _ => rejectWildcardType(infixType()) @@ -2904,7 +2770,7 @@ object Parsers { } CaseDef(pat, EmptyTree, atSpan(accept(ARROW)) { val t = indentedRegionAfterArrow(rejectWildcardType(typ()), inCaseDef = true) - if in.token == SEMI then nextToken() + if in.token == SEMI then in.nextToken() newLinesOptWhenFollowedBy(CASE) t }) @@ -2921,7 +2787,7 @@ object Parsers { else pat def patternAlts(location: Location): List[Tree] = - if (isIdent(nme.raw.BAR)) { nextToken(); pattern1(location) :: patternAlts(location) } + if (isIdent(nme.raw.BAR)) { in.nextToken(); pattern1(location) :: patternAlts(location) } else Nil /** Pattern1 ::= PatVar Ascription @@ -2943,7 +2809,7 @@ object Parsers { warnFrom = `3.3`, errorFrom = future ) - nextToken() + in.nextToken() ascription(p, location) else p @@ -2953,7 +2819,7 @@ object Parsers { def pattern3(): Tree = val p = infixPattern() if followingIsVararg() then - val start = skipToken() + val start = in.skipToken() p match case p @ Ident(name) if name.isVarPattern => Typed(p, atSpan(start) { Ident(tpnme.WILDCARD_STAR) }) @@ -2966,7 +2832,7 @@ object Parsers { */ val pattern2: () => Tree = () => pattern3() match case p @ Ident(name) if in.token == AT => - val offset = skipToken() + val offset = in.skipToken() pattern3() match { case pt @ Bind(nme.WILDCARD, pt1: Typed) if pt.mods.is(Given) => atSpan(startOffset(p), 0) { Bind(name, pt1).withMods(pt.mods) } @@ -3017,7 +2883,7 @@ object Parsers { xmlLiteralPattern() case GIVEN => atSpan(in.offset) { - val givenMod = atSpan(skipToken())(Mod.Given()) + val givenMod = atSpan(in.skipToken())(Mod.Given()) val typed = Typed(Ident(nme.WILDCARD), refinedType()) Bind(nme.WILDCARD, typed).withMods(addMod(Modifiers(), givenMod)) } @@ -3034,7 +2900,7 @@ object Parsers { def simplePatternRest(t: Tree): Tree = if in.token == DOT then - nextToken() + in.nextToken() simplePatternRest(selector(t)) else var p = t @@ -3095,7 +2961,7 @@ object Parsers { private def addModifier(mods: Modifiers): Modifiers = { val tok = in.token val name = in.name - val mod = atSpan(skipToken()) { modOfToken(tok, name) } + val mod = atSpan(in.skipToken()) { modOfToken(tok, name) } if (mods.isOneOf(mod.flags)) syntaxError(RepeatedModifier(mod.flags.flagsString)) addMod(mods, mod) @@ -3120,7 +2986,7 @@ object Parsers { if sourceVersion.isAtLeast(future) then deprecationWarning( em"The [this] qualifier will be deprecated in the future; it should be dropped.") - nextToken() + in.nextToken() mods | Local else mods.withPrivateWithin(ident().toTypeName) } @@ -3149,7 +3015,7 @@ object Parsers { val mods1 = addModifier(mods) loop(if (isAccessMod) accessQualifierOpt(mods1) else mods1) else if (in.isNewLine && (mods.hasFlags || mods.hasAnnotations)) { - nextToken() + in.nextToken() loop(mods) } else @@ -3249,7 +3115,7 @@ object Parsers { def checkVarianceOK(): Boolean = val ok = ownerKind != ParamOwner.Def && ownerKind != ParamOwner.TypeParam if !ok then syntaxError(em"no `+/-` variance annotation allowed here") - nextToken() + in.nextToken() ok def typeParam(): TypeDef = { @@ -3260,7 +3126,7 @@ object Parsers { if Feature.ccEnabled && in.token == SEALED then if ownerKind == ParamOwner.Def then mods |= Sealed else syntaxError(em"`sealed` modifier only allowed for method type parameters") - nextToken() + in.nextToken() if isIdent(nme.raw.PLUS) && checkVarianceOK() then mods |= Covariant else if isIdent(nme.raw.MINUS) && checkVarianceOK() then @@ -3268,7 +3134,7 @@ object Parsers { atSpan(start, nameStart) { val name = if (isAbstractOwner && in.token == USCORE) { - nextToken() + in.nextToken() WildcardParamName.fresh().toTypeName } else ident().toTypeName @@ -3320,7 +3186,7 @@ object Parsers { ): List[ValDef] = { var impliedMods: Modifiers = EmptyModifiers - def addParamMod(mod: () => Mod) = impliedMods = addMod(impliedMods, atSpan(skipToken()) { mod() }) + def addParamMod(mod: () => Mod) = impliedMods = addMod(impliedMods, atSpan(in.skipToken()) { mod() }) def paramMods() = if in.token == IMPLICIT then @@ -3338,10 +3204,10 @@ object Parsers { mods = addFlag(modifiers(start = mods), ParamAccessor) mods = if in.token == VAL then - nextToken() + in.nextToken() mods else if in.token == VAR then - val mod = atSpan(skipToken()) { Mod.Var() } + val mod = atSpan(in.skipToken()) { Mod.Var() } addMod(mods, mod) else if (!(mods.flags &~ (ParamAccessor | Inline | Erased | impliedMods.flags)).isEmpty) @@ -3362,7 +3228,7 @@ object Parsers { // needed?, it's checked later anyway val tpt = paramType() val default = - if (in.token == EQUALS) { nextToken(); subExpr() } + if (in.token == EQUALS) { in.nextToken(); subExpr() } else EmptyTree if (impliedMods.mods.nonEmpty) impliedMods = impliedMods.withMods(Nil) // keep only flags, so that parameter positions don't overlap @@ -3522,12 +3388,12 @@ object Parsers { in.sourcePos(), from = future) patch(source, Span(in.offset, in.offset + 1), "*") - ImportSelector(atSpan(skipToken()) { Ident(nme.WILDCARD) }) + ImportSelector(atSpan(in.skipToken()) { Ident(nme.WILDCARD) }) /** 'given [InfixType]' */ def givenSelector() = ImportSelector( - atSpan(skipToken()) { Ident(nme.EMPTY) }, + atSpan(in.skipToken()) { Ident(nme.EMPTY) }, bound = if canStartInfixTypeTokens.contains(in.token) then rejectWildcardType(infixType()) else EmptyTree) @@ -3543,7 +3409,7 @@ object Parsers { patch(source, Span(in.offset, in.offset + 2), if testChar(in.offset - 1, ' ') && testChar(in.offset + 2, ' ') then "as" else " as ") - atSpan(startOffset(from), skipToken()) { + atSpan(startOffset(from), in.skipToken()) { val to = if in.token == USCORE then wildcardIdent() else termIdent() ImportSelector(from, if to.name == nme.ERROR then EmptyTree else to) } @@ -3614,16 +3480,16 @@ object Parsers { */ def defOrDcl(start: Int, mods: Modifiers): Tree = in.token match { case VAL => - nextToken() + in.nextToken() patDefOrDcl(start, mods) case VAR => - val mod = atSpan(skipToken()) { Mod.Var() } + val mod = atSpan(in.skipToken()) { Mod.Var() } val mod1 = addMod(mods, mod) patDefOrDcl(start, mod1) case DEF => - defDefOrDcl(start, skipToken(mods)) + defDefOrDcl(start, in.skipToken(mods)) case TYPE => - typeDefOrDcl(start, skipToken(mods)) + typeDefOrDcl(start, in.skipToken(mods)) case CASE if inEnum => enumCase(start, mods) case _ => @@ -3641,7 +3507,7 @@ object Parsers { val first = pattern2() var lhs = first match { case id: Ident if in.token == COMMA => - nextToken() + in.nextToken() id :: commaSeparated(() => termIdent()) case _ => first :: Nil @@ -3708,7 +3574,7 @@ object Parsers { false if (in.token == THIS) { - nextToken() + in.nextToken() val vparamss = termParamClauses(numLeadParams = numLeadParams) if (vparamss.isEmpty || vparamss.head.take(1).exists(_.mods.isOneOf(GivenOrImplicit))) in.token match { @@ -3742,7 +3608,7 @@ object Parsers { if (migrateTo3) newLineOptWhenFollowedBy(LBRACE) val rhs = if in.token == EQUALS then - nextToken() + in.nextToken() subExpr() else if !tpt.isEmpty then EmptyTree @@ -3769,7 +3635,7 @@ object Parsers { atSpan(in.offset) { inBracesOrIndented { val stats = selfInvocation() :: ( - if (isStatSep) { nextToken(); blockStatSeq() } + if (isStatSep) { in.nextToken(); blockStatSeq() } else Nil) Block(stats, Literal(Constant(()))) } @@ -3801,12 +3667,12 @@ object Parsers { } in.token match { case EQUALS => - nextToken() + in.nextToken() makeTypeDef(toplevelTyp()) case SUBTYPE | SUPERTYPE => val bounds = typeBounds() if (in.token == EQUALS) { - val eqOffset = skipToken() + val eqOffset = in.skipToken() var rhs = toplevelTyp() rhs match { case mtt: MatchTypeTree => @@ -3842,19 +3708,19 @@ object Parsers { def tmplDef(start: Int, mods: Modifiers): Tree = in.token match { case TRAIT => - classDef(start, skipToken(addFlag(mods, Trait))) + classDef(start, in.skipToken(addFlag(mods, Trait))) case CLASS => - classDef(start, skipToken(mods)) + classDef(start, in.skipToken(mods)) case CASECLASS => - classDef(start, skipToken(mods | Case)) + classDef(start, in.skipToken(mods | Case)) case OBJECT => - objectDef(start, skipToken(mods | Module)) + objectDef(start, in.skipToken(mods | Module)) case CASEOBJECT => - objectDef(start, skipToken(mods | Case | Module)) + objectDef(start, in.skipToken(mods | Case | Module)) case ENUM => - enumDef(start, skipToken(mods | Enum)) + enumDef(start, in.skipToken(mods | Enum)) case GIVEN => - givenDef(start, mods, atSpan(skipToken()) { Mod.Given() }) + givenDef(start, mods, atSpan(in.skipToken()) { Mod.Given() }) case _ => val start = in.lastOffset syntaxErrorOrIncomplete(ExpectedStartOfTopLevelDefinition()) @@ -3924,7 +3790,7 @@ object Parsers { atSpan(start, nameStart) { val id = termIdent() if (in.token == COMMA) { - nextToken() + in.nextToken() val ids = commaSeparated(() => termIdent()) PatDef(mods1, id :: ids, TypeTree(), EmptyTree) } @@ -3946,7 +3812,7 @@ object Parsers { def caseTemplate(constr: DefDef): Template = { val parents = if (in.token == EXTENDS) { - nextToken() + in.nextToken() constrApps() } else Nil @@ -4022,7 +3888,7 @@ object Parsers { * {UsingParamClause} ExtMethods */ def extension(): ExtMethods = - val start = skipToken() + val start = in.skipToken() val tparams = typeParamClauseOpt(ParamOwner.Def) val leadParamss = ListBuffer[List[ValDef]]() def numLeadParams = leadParamss.map(_.length).sum @@ -4034,7 +3900,7 @@ object Parsers { leadParamss ++= termParamClauses(givenOnly = true, numLeadParams = numLeadParams) if in.isColon then syntaxError(em"no `:` expected here") - nextToken() + in.nextToken() val methods: List[Tree] = if in.token == EXPORT then exportClause() @@ -4098,7 +3964,7 @@ object Parsers { val ts = val tok = in.token if (tok == WITH || tok == COMMA) && tok != exclude then - nextToken() + in.nextToken() constrApps(exclude = if tok == WITH then COMMA else WITH) else Nil t :: ts @@ -4110,7 +3976,7 @@ object Parsers { val la = in.lookahead la.isAfterLineEnd || la.token == LBRACE if in.token == WITH && !isTemplateStart then - nextToken() + in.nextToken() constrApp() :: withConstrApps() else Nil @@ -4120,7 +3986,7 @@ object Parsers { def template(constr: DefDef, isEnum: Boolean = false): Template = { val parents = if (in.token == EXTENDS) { - nextToken() + in.nextToken() if (in.token == LBRACE || in.token == COLONeol) { report.errorOrMigrationWarning( em"`extends` must be followed by at least one parent", @@ -4133,7 +3999,7 @@ object Parsers { newLinesOptWhenFollowedBy(nme.derives) val derived = if (isIdent(nme.derives)) { - nextToken() + in.nextToken() commaSeparated(() => convertToTypeId(qualId())) } else Nil @@ -4175,7 +4041,7 @@ object Parsers { val r = inDefScopeBraces(templateStatSeq(), rewriteWithColon) if in.token == WITH && parents.isEmpty then syntaxError(EarlyDefinitionsNotSupported()) - nextToken() + in.nextToken() template(emptyConstructor) r @@ -4215,9 +4081,9 @@ object Parsers { while var empty = false if (in.token == PACKAGE) { - val start = skipToken() + val start = in.skipToken() if (in.token == OBJECT) { - nextToken() + in.nextToken() stats += objectDef(start, Modifiers(Package)) } else stats += packaging(start) @@ -4248,19 +4114,19 @@ object Parsers { atSpan(in.offset) { val selfName = if in.token == THIS then - nextToken() + in.nextToken() nme.WILDCARD else ident() val selfTpt = if in.isColon then - nextToken() + in.nextToken() infixType() else if selfName == nme.WILDCARD then accept(COLONfollow) TypeTree() if in.token == ARROW then in.token = SELFARROW // suppresses INDENT insertion after `=>` - nextToken() + in.nextToken() else syntaxError(em"`=>` expected after self type") makeSelfDef(selfName, selfTpt) @@ -4390,12 +4256,12 @@ object Parsers { def compilationUnit(): Tree = checkNoEscapingPlaceholders { def topstats(): List[Tree] = { val ts = new ListBuffer[Tree] - while (in.token == SEMI) nextToken() + while (in.token == SEMI) in.nextToken() val start = in.offset if (in.token == PACKAGE) { - nextToken() + in.nextToken() if (in.token == OBJECT) { - nextToken() + in.nextToken() ts += objectDef(start, Modifiers(Package)) if (in.token != EOF) { statSepOrEnd(ts, what = "toplevel definition") @@ -4450,4 +4316,139 @@ object Parsers { (EmptyValDef, List(EmptyTree)) } } + + /** The Scala parser that can rewrite to indent */ + class ToIndentParser(source: SourceFile)(using Context) extends Parser(source): + class ToIndentScanner(source: SourceFile)(using Context) extends Scanner(source): + /** A copy of the previous token */ + var prev: TokenData = Scanners.newTokenData + + override def nextToken(): Unit = + if token != EMPTY then patchIndent() + prev = saveCopy + super.nextToken() + end ToIndentScanner + + override val in: ToIndentScanner = new ToIndentScanner(source) + assert(in.rewriteToIndent) + + /** The last offset where a colon at the end of line would be required if a subsequent { ... } + * block would be converted to an indentation region. */ + private var possibleColonOffset: Int = -1 + + /** the minimum indent width to rewrite to */ + private var minimumIndent: IndentWidth = IndentWidth.Zero + + /** the maximum indent width to ensure an indent region is properly closed by outdentation */ + private var maximumIndent: Option[IndentWidth] = None + + override def colonAtEOLOpt(): Unit = + possibleColonOffset = in.lastOffset + super.colonAtEOLOpt() + + /** make sure there is an indent after a `=>\n` */ + override def indentedRegionAfterArrow[T](body: => T, inCaseDef: Boolean = false): T = + if in.rewriteToIndent && (inCaseDef || in.isAfterLineEnd) then toIndentedRegion(body) + else body + + override def inBracesOrIndented[T](body: => T, rewriteWithColon: Boolean = false): T = + if in.token == INDENT then + if isAfterArrow then enclosed(INDENT, body) + else enclosed(INDENT, toIndentedRegion(body)) + else + if isAfterArrow then + val t = enclosed(LBRACE, toIndentedRegion(body)) + maximumIndent = None // no need to force outdentation after `}` + t + else bracesToIndented(body, rewriteWithColon) + + private val bracesToIndentPredecessors = + colonEOLPredecessors | canStartIndentTokens | BitSet(IDENTIFIER) + + /** Parse brace-enclosed `body` and rewrite it to be an indentation region instead, if possible. + * If possible means: + * 1. not inside (...), [...], case ... => + * 2. opening brace `{` is at end of line + * 3. closing brace `}` is at start of line + * 4. there is at least one token between the braces + * 5. the closing brace is also at the end of the line, or it is followed by one of + * `then`, `else`, `do`, `catch`, `finally`, `yield`, or `match`. + * 6. the opening brace `}` follows a colonEOLPredecessors, a canStartIndentTokens or an + * identifier + * 7. last token is not a leading operator + */ + private def bracesToIndented[T](body: => T, rewriteWithColon: Boolean): T = + val prevSaved = in.prev.saveCopy + val lastOffsetSaved = in.lastOffset + val underColonSyntax = possibleColonOffset == in.lastOffset + val colonRequired = rewriteWithColon || underColonSyntax + val (startOpening, endOpening) = elimRegion(in.offset) + def isBracesOrIndented(r: Region): Boolean = r match + case r: (Indented | InBraces) => true + case _ => false + var canRewrite = + isBracesOrIndented(in.currentRegion) // test (1) + && bracesToIndentPredecessors.contains(prevSaved.token) // test (6) + && !(prevSaved.isOperator && prevSaved.isAfterLineEnd) // test (7) + val t = enclosed(LBRACE, { + if in.isAfterLineEnd && in.token != RBRACE then // test (2)(4) + toIndentedRegion: + try body + finally canRewrite &= in.isAfterLineEnd // test (3) + else + canRewrite = false + body + }) + canRewrite &= in.isAfterLineEnd || in.token == EOF || statCtdTokens.contains(in.token) // test (5) + if canRewrite && (!underColonSyntax || Feature.fewerBracesEnabled) then + val (startClosing, endClosing) = elimRegion(in.lastOffset - 1) + // patch over the added indentation to remove braces + patchOver(source, Span(startOpening, endOpening), "") + patchOver(source, Span(startClosing, endClosing), "") + if colonRequired then + if prevSaved.token == IDENTIFIER && prevSaved.isOperator then + patch(Span(prevSaved.offset, lastOffsetSaved), s"`${prevSaved.name}`:") + else if prevSaved.token == IDENTIFIER && prevSaved.name.last == '_' then + patch(Span(lastOffsetSaved), " :") + else patch(Span(lastOffsetSaved), ":") + else + // no need to force outdentation after `}` + maximumIndent = None + t + end bracesToIndented + + /** compute required indentation to indent region properly */ + private def toIndentedRegion[T](body: => T): T = + val enclosingIndent = minimumIndent + minimumIndent = + if enclosingIndent < in.currentRegion.indentWidth then + in.currentRegion.indentWidth + else if + in.token == CASE && ( + in.currentRegion.enclosing == null || + in.currentRegion.indentWidth == in.currentRegion.enclosing.indentWidth + ) + then enclosingIndent + else enclosingIndent.increment + try body + finally + maximumIndent = Some(minimumIndent) + minimumIndent = enclosingIndent + + /** check that indentaton is correct or patch */ + private def patchIndent(): Unit = + if in.isAfterLineEnd && !in.isNewLine && in.token != OUTDENT && in.token != INDENT then + val currentIndent = in.indentWidth(in.offset) + val indentEndOffset = in.lineOffset + currentIndent.size + def isDotOrClosing = (closingParens + DOT).contains(in.token) + val needsOutdent = maximumIndent.exists: max => + currentIndent >= max || (!isDotOrClosing && currentIndent > minimumIndent) + val offByOne = + currentIndent != minimumIndent && currentIndent.isClose(minimumIndent) + if needsOutdent || !(currentIndent >= minimumIndent) || offByOne then + patch(Span(in.lineOffset, indentEndOffset), minimumIndent.toPrefix) + // no need to outdent anymore + if in.token != RBRACE then + maximumIndent = None + end ToIndentParser } diff --git a/compiler/src/dotty/tools/dotc/parsing/ScriptParsers.scala b/compiler/src/dotty/tools/dotc/parsing/ScriptParsers.scala deleted file mode 100644 index d11db73b0455..000000000000 --- a/compiler/src/dotty/tools/dotc/parsing/ScriptParsers.scala +++ /dev/null @@ -1,147 +0,0 @@ -package dotty.tools -package dotc -package parsing - -import util.SourceFile -import core._ -import Contexts._ -import Parsers._ - - -/**

Performs the following context-free rewritings:

- *
    - *
  1. - * Places all pattern variables in Bind nodes. In a pattern, for - * identifiers x:
    - *                 x  => x @ _
    - *               x:T  => x @ (_ : T)
    - *
  2. - *
  3. Removes pattern definitions (PatDef's) as follows: - * If pattern is a simple (typed) identifier:
    - *        val x = e     ==>  val x = e
    - *        val x: T = e  ==>  val x: T = e
    - * - * if there are no variables in pattern
    - *        val p = e  ==>  e match (case p => ())
    - * - * if there is exactly one variable in pattern
    - *        val x_1 = e match (case p => (x_1))
    - * - * if there is more than one variable in pattern
    - *        val p = e  ==>  private synthetic val t$ = e match (case p => (x_1, ..., x_N))
    - *                        val x_1 = t$._1
    - *                        ...
    - *                        val x_N = t$._N
    - *
  4. - *
  5. - * Removes function types as follows:
    - *        (argtpes) => restpe   ==>   scala.Function_n[argtpes, restpe]
    - *
  6. - *
  7. - * Wraps naked case definitions in a match as follows:
    - *        { cases }   ==>   (x => x.match {cases}), except when already argument to match
    - *
  8. - *
- */ -object ScriptParsers { - - import ast.untpd._ - - class ScriptParser(source: SourceFile)(using Context) extends Parser(source) { - - /** This is the parse entry point for code which is not self-contained, e.g. - * a script which is a series of template statements. They will be - * swaddled in Trees until the AST is equivalent to the one returned - * by compilationUnit(). - */ - override def parse(): Tree = unsupported("parse") - } -} - - /* TODO: reinstantiate - - val stmts = templateStatSeq(false)._2 - accept(EOF) - - def mainModuleName = ctx.settings.script.value - - /** If there is only a single object template in the file and it has a - * suitable main method, we will use it rather than building another object - * around it. Since objects are loaded lazily the whole script would have - * been a no-op, so we're not taking much liberty. - */ - def searchForMain(): Option[Tree] = { - /** Have to be fairly liberal about what constitutes a main method since - * nothing has been typed yet - for instance we can't assume the parameter - * type will look exactly like "Array[String]" as it could have been renamed - * via import, etc. - */ - def isMainMethod(t: Tree) = t match { - case DefDef(_, nme.main, Nil, List(_), _, _) => true - case _ => false - } - /** For now we require there only be one top level object. */ - var seenModule = false - val newStmts = stmts collect { - case t @ Import(_, _) => t - case md @ ModuleDef(mods, name, template) - if !seenModule && (template.body exists isMainMethod) => - seenModule = true - /** This slightly hacky situation arises because we have no way to communicate - * back to the scriptrunner what the name of the program is. Even if we were - * willing to take the sketchy route of settings.script.value = progName, that - * does not work when using fsc. And to find out in advance would impose a - * whole additional parse. So instead, if the actual object's name differs from - * what the script is expecting, we transform it to match. - */ - md.derivedModuleDef(mods, mainModuleName.toTermName, template) - case _ => - /** If we see anything but the above, fail. */ - return None - } - Some(makePackaging(0, emptyPkg, newStmts)) - } - - if (mainModuleName == ScriptRunner.defaultScriptMain) - searchForMain() foreach { return _ } - - /** Here we are building an AST representing the following source fiction, - * where is from -Xscript (defaults to "Main") and are - * the result of parsing the script file. - * - * object { - * def main(argv: Array[String]): Unit = { - * val args = argv - * new AnyRef { - * - * } - * } - * } - */ - import definitions._ - - def emptyPkg = atPos(0, 0, 0) { Ident(nme.EMPTY_PACKAGE_NAME) } - def emptyInit = DefDef( - Modifiers(), - nme.CONSTRUCTOR, - Nil, - List(Nil), - TypeTree(), - Block(List(Apply(Select(Super(This(tpnme.EMPTY), tpnme.EMPTY), nme.CONSTRUCTOR), Nil)), Literal(Constant(()))) - ) - - // def main - def mainParamType = AppliedTypeTree(Ident(tpnme.Array), List(Ident(tpnme.String))) - def mainParameter = List(ValDef(Modifiers(Param), "argv", mainParamType, EmptyTree)) - def mainSetArgv = List(ValDef(Modifiers(), "args", TypeTree(), Ident("argv"))) - def mainNew = makeNew(Nil, emptyValDef, stmts, List(Nil), NoPosition, NoPosition) - def mainDef = DefDef(Modifiers(), nme.main, Nil, List(mainParameter), scalaDot(tpnme.Unit), Block(mainSetArgv, mainNew)) - - // object Main - def moduleName = ScriptRunner scriptMain settings - def moduleBody = Template(List(scalaScalaObjectConstr), emptyValDef, List(emptyInit, mainDef)) - def moduleDef = ModuleDef(Modifiers(), moduleName, moduleBody) - - // package { ... } - makePackaging(0, emptyPkg, List(moduleDef)) - }*/ diff --git a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala index 53e6b9472f5e..73b86ad42d35 100644 --- a/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala +++ b/compiler/src/dotty/tools/dotc/printing/SyntaxHighlighting.scala @@ -6,7 +6,7 @@ import scala.language.unsafeNulls import dotty.tools.dotc.ast.untpd import dotty.tools.dotc.core.Contexts._ import dotty.tools.dotc.core.StdNames._ -import dotty.tools.dotc.parsing.Parsers.Parser +import dotty.tools.dotc.parsing.Parsers import dotty.tools.dotc.parsing.Scanners.Scanner import dotty.tools.dotc.parsing.Tokens._ import dotty.tools.dotc.reporting.Reporter @@ -124,7 +124,7 @@ object SyntaxHighlighting { } } - val parser = new Parser(source) + val parser = Parsers.parser(source) val trees = parser.blockStatSeq() TreeHighlighter.highlight(trees) diff --git a/compiler/src/dotty/tools/dotc/typer/Namer.scala b/compiler/src/dotty/tools/dotc/typer/Namer.scala index df708057dd71..d18e04e09934 100644 --- a/compiler/src/dotty/tools/dotc/typer/Namer.scala +++ b/compiler/src/dotty/tools/dotc/typer/Namer.scala @@ -19,7 +19,7 @@ import config.{Config, Feature} import config.Printers.typr import inlines.{Inlines, PrepareInlineable} import parsing.JavaParsers.JavaParser -import parsing.Parsers.Parser +import parsing.Parsers import Annotations._ import Inferencing._ import transform.ValueClasses._ @@ -739,7 +739,7 @@ class Namer { typer: Typer => unit.untpdTree = if (unit.isJava) new JavaParser(unit.source).parse() - else new Parser(unit.source).parse() + else Parsers.parser(unit.source).parse() atPhase(Phases.typerPhase) { inContext(PrepareInlineable.initContext(ctx)) { diff --git a/compiler/src/dotty/tools/repl/ParseResult.scala b/compiler/src/dotty/tools/repl/ParseResult.scala index a67b247066f7..0130150496b1 100644 --- a/compiler/src/dotty/tools/repl/ParseResult.scala +++ b/compiler/src/dotty/tools/repl/ParseResult.scala @@ -5,7 +5,7 @@ import dotc.CompilationUnit import dotc.ast.untpd import dotc.core.Contexts._ import dotc.core.StdNames.str -import dotc.parsing.Parsers.Parser +import dotc.parsing.Parsers import dotc.parsing.Tokens import dotc.reporting.{Diagnostic, StoreReporter} import dotc.util.SourceFile @@ -121,7 +121,7 @@ object ParseResult { @sharable private val CommandExtract = """(:[\S]+)\s*(.*)""".r private def parseStats(using Context): List[untpd.Tree] = { - val parser = new Parser(ctx.source) + val parser = Parsers.parser(ctx.source) val stats = parser.blockStatSeq() parser.accept(Tokens.EOF) stats diff --git a/compiler/test/dotty/tools/dotc/ast/UntypedTreeMapTest.scala b/compiler/test/dotty/tools/dotc/ast/UntypedTreeMapTest.scala index 472e47d859f3..f5ba121308d8 100644 --- a/compiler/test/dotty/tools/dotc/ast/UntypedTreeMapTest.scala +++ b/compiler/test/dotty/tools/dotc/ast/UntypedTreeMapTest.scala @@ -6,7 +6,7 @@ import org.junit.Test import org.junit.Assert._ import dotc.core.Contexts._ -import dotc.parsing.Parsers.Parser +import dotc.parsing.Parsers import dotc.util.SourceFile class UntpdTreeMapTest extends DottyTest { @@ -14,7 +14,7 @@ class UntpdTreeMapTest extends DottyTest { import untpd._ def parse(code: String): Tree = { - val (_, stats) = new Parser(SourceFile.virtual("", code)).templateStatSeq() + val (_, stats) = Parsers.parser(SourceFile.virtual("", code)).templateStatSeq() stats match { case List(stat) => stat; case stats => untpd.Thicket(stats) } } From de7633598d1333c00f72bc3bae3f5be5bd1daad6 Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Mon, 19 Jun 2023 17:00:53 +0200 Subject: [PATCH 8/9] Silence initialization checker --- .../dotty/tools/dotc/parsing/Parsers.scala | 19 ++++++++++--------- .../dotty/tools/dotc/parsing/Scanners.scala | 2 +- .../dotc/parsing/ModifiersParsingTest.scala | 2 +- .../dotty/tools/dotc/parsing/ParserTest.scala | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala index 70969491e758..0ba8bcad9da5 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Parsers.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Parsers.scala @@ -168,11 +168,14 @@ object Parsers { } } - class Parser private[parsing] (source: SourceFile, allowRewrite: Boolean = true)(using Context) extends ParserCommon(source) { + class Parser private[Parsers] (source: SourceFile, allowRewrite: Boolean = true)(using Context) extends ParserCommon(source) { - val in: Scanner = new Scanner(source, profile = Profile.current, allowRewrite = allowRewrite) + val in: Scanner = createScanner() // in.debugTokenStream = true // uncomment to see the token stream of the standard scanner, but not syntax highlighting + def createScanner() = + new Scanner(source, profile = Profile.current, allowRewrite = allowRewrite) + /** This is the general parse entry point. */ def parse(): Tree = { @@ -4318,19 +4321,17 @@ object Parsers { } /** The Scala parser that can rewrite to indent */ - class ToIndentParser(source: SourceFile)(using Context) extends Parser(source): - class ToIndentScanner(source: SourceFile)(using Context) extends Scanner(source): - /** A copy of the previous token */ - var prev: TokenData = Scanners.newTokenData + private class ToIndentParser(source: SourceFile)(using Context) extends Parser(source): + override def createScanner(): Scanner = new Scanner(source): override def nextToken(): Unit = if token != EMPTY then patchIndent() prev = saveCopy super.nextToken() - end ToIndentScanner - override val in: ToIndentScanner = new ToIndentScanner(source) assert(in.rewriteToIndent) + + private var prev: TokenData = Scanners.newTokenData /** The last offset where a colon at the end of line would be required if a subsequent { ... } * block would be converted to an indentation region. */ @@ -4378,7 +4379,7 @@ object Parsers { * 7. last token is not a leading operator */ private def bracesToIndented[T](body: => T, rewriteWithColon: Boolean): T = - val prevSaved = in.prev.saveCopy + val prevSaved = prev.saveCopy val lastOffsetSaved = in.lastOffset val underColonSyntax = possibleColonOffset == in.lastOffset val colonRequired = rewriteWithColon || underColonSyntax diff --git a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala index ef95f6b7be5e..7eeb1fc8b0a7 100644 --- a/compiler/src/dotty/tools/dotc/parsing/Scanners.scala +++ b/compiler/src/dotty/tools/dotc/parsing/Scanners.scala @@ -1553,7 +1553,7 @@ object Scanners { /* Initialization: read first char, then first token */ nextChar() - nextToken() + (this: @unchecked).nextToken() currentRegion = topLevelRegion(indentWidth(offset)) } end Scanner diff --git a/compiler/test/dotty/tools/dotc/parsing/ModifiersParsingTest.scala b/compiler/test/dotty/tools/dotc/parsing/ModifiersParsingTest.scala index 15d43653025a..65d291245df1 100644 --- a/compiler/test/dotty/tools/dotc/parsing/ModifiersParsingTest.scala +++ b/compiler/test/dotty/tools/dotc/parsing/ModifiersParsingTest.scala @@ -17,7 +17,7 @@ object ModifiersParsingTest { given Context = (new ContextBase).initialCtx def parse(code: String): Tree = { - val (_, stats) = new Parser(SourceFile.virtual("", code)).templateStatSeq() + val (_, stats) = Parsers.parser(SourceFile.virtual("", code)).templateStatSeq() stats match { case List(stat) => stat; case stats => Thicket(stats) } } diff --git a/compiler/test/dotty/tools/dotc/parsing/ParserTest.scala b/compiler/test/dotty/tools/dotc/parsing/ParserTest.scala index 4b166f838d5e..ebf119a6930c 100644 --- a/compiler/test/dotty/tools/dotc/parsing/ParserTest.scala +++ b/compiler/test/dotty/tools/dotc/parsing/ParserTest.scala @@ -25,7 +25,7 @@ class ParserTest extends DottyTest { private def parseSource(source: SourceFile): Tree = { //println("***** parsing " + source.file) - val parser = new Parser(source) + val parser = Parsers.parser(source) val tree = parser.parse() parsed += 1 parsedTrees += tree From f4b91e310a0841917ab7039784a16fc47774f4f7 Mon Sep 17 00:00:00 2001 From: Adrien Piquerez Date: Wed, 5 Jul 2023 11:25:21 +0200 Subject: [PATCH 9/9] Fix indent-comments test We cannot remove braces after a new line, to prevent from removing braces in stat seq. --- tests/rewrites/indent-comments.check | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/rewrites/indent-comments.check b/tests/rewrites/indent-comments.check index 38290f23e8fa..8b5bec769366 100644 --- a/tests/rewrites/indent-comments.check +++ b/tests/rewrites/indent-comments.check @@ -2,9 +2,9 @@ class A: /* 1 */ /* 2 */ def m1(b: Boolean) = /* 3 */ /* 4 */ val x = if (b) - /* 5 */ + /* 5 */ { "true" - /* 6 */ + } /* 6 */ else /* 7 */ "false"