diff --git a/crates/hir-def/src/macro_expansion_tests/mbe.rs b/crates/hir-def/src/macro_expansion_tests/mbe.rs index ddf1a213d7bd..f99030950dd3 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe.rs @@ -1979,3 +1979,51 @@ fn f() { "#]], ); } + +#[test] +fn semicolon_does_not_glue() { + check( + r#" +macro_rules! bug { + ($id: expr) => { + true + }; + ($id: expr; $($attr: ident),*) => { + true + }; + ($id: expr; $($attr: ident),*; $norm: expr) => { + true + }; + ($id: expr; $($attr: ident),*;; $print: expr) => { + true + }; + ($id: expr; $($attr: ident),*; $norm: expr; $print: expr) => { + true + }; +} + +let _ = bug!(a;;;test); + "#, + expect![[r#" +macro_rules! bug { + ($id: expr) => { + true + }; + ($id: expr; $($attr: ident),*) => { + true + }; + ($id: expr; $($attr: ident),*; $norm: expr) => { + true + }; + ($id: expr; $($attr: ident),*;; $print: expr) => { + true + }; + ($id: expr; $($attr: ident),*; $norm: expr; $print: expr) => { + true + }; +} + +let _ = true; + "#]], + ); +} diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs b/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs index 1bbed01443de..cb4fcd887d8a 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs @@ -582,8 +582,8 @@ macro_rules! arbitrary { } impl $crate::arbitrary::Arbitrary for Vec { - type Parameters = RangedParams1; - type Strategy = VecStrategy; + type Parameters = RangedParams1 ; + type Strategy = VecStrategy ; fn arbitrary_with(args: Self::Parameters) -> Self::Strategy { { let product_unpack![range, a] = args; vec(any_with::(a), range) diff --git a/crates/ide/src/expand_macro.rs b/crates/ide/src/expand_macro.rs index 4811f1f691c8..77668973ea14 100644 --- a/crates/ide/src/expand_macro.rs +++ b/crates/ide/src/expand_macro.rs @@ -677,4 +677,26 @@ crate::Foo; crate::Foo;"#]], ); } + + #[test] + fn semi_glueing() { + check( + r#" +macro_rules! __log_value { + ($key:ident :$capture:tt =) => {}; +} + +macro_rules! __log { + ($key:tt $(:$capture:tt)? $(= $value:expr)?; $($arg:tt)+) => { + __log_value!($key $(:$capture)* = $($value)*); + }; +} + +__log!(written:%; "Test"$0); + "#, + expect![[r#" + __log! + "#]], + ); + } } diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index b1f542eac7ce..f3f9f2990c22 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -389,8 +389,13 @@ fn expand_var( match ctx.bindings.get_fragment(v, id, &mut ctx.nesting, marker) { Ok(fragment) => { match fragment { - Fragment::Tokens(tt) => builder.extend_with_tt(tt.strip_invisible()), - Fragment::TokensOwned(tt) => builder.extend_with_tt(tt.view().strip_invisible()), + // rustc spacing is not like ours. Ours is like proc macros', it dictates how puncts will actually be joined. + // rustc uses them mostly for pretty printing. So we have to deviate a bit from what rustc does here. + // Basically, a metavariable can never be joined with whatever after it. + Fragment::Tokens(tt) => builder.extend_with_tt_alone(tt.strip_invisible()), + Fragment::TokensOwned(tt) => { + builder.extend_with_tt_alone(tt.view().strip_invisible()) + } Fragment::Expr(sub) => { let sub = sub.strip_invisible(); let mut span = id; @@ -402,7 +407,7 @@ fn expand_var( if wrap_in_parens { builder.open(tt::DelimiterKind::Parenthesis, span); } - builder.extend_with_tt(sub); + builder.extend_with_tt_alone(sub); if wrap_in_parens { builder.close(span); } diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs index 7be49cbc7e11..8a2f12421394 100644 --- a/crates/mbe/src/parser.rs +++ b/crates/mbe/src/parser.rs @@ -6,7 +6,10 @@ use std::sync::Arc; use arrayvec::ArrayVec; use intern::{Symbol, sym}; use span::{Edition, Span, SyntaxContext}; -use tt::iter::{TtElement, TtIter}; +use tt::{ + MAX_GLUED_PUNCT_LEN, + iter::{TtElement, TtIter}, +}; use crate::ParseError; @@ -96,7 +99,7 @@ pub(crate) enum Op { delimiter: tt::Delimiter, }, Literal(tt::Literal), - Punct(Box, 3>>), + Punct(Box, MAX_GLUED_PUNCT_LEN>>), Ident(tt::Ident), } @@ -151,7 +154,7 @@ pub(crate) enum MetaVarKind { pub(crate) enum Separator { Literal(tt::Literal), Ident(tt::Ident), - Puncts(ArrayVec, 3>), + Puncts(ArrayVec, MAX_GLUED_PUNCT_LEN>), } // Note that when we compare a Separator, we just care about its textual value. diff --git a/crates/tt/src/iter.rs b/crates/tt/src/iter.rs index 1d88218810de..0418c00174bd 100644 --- a/crates/tt/src/iter.rs +++ b/crates/tt/src/iter.rs @@ -6,7 +6,7 @@ use std::fmt; use arrayvec::ArrayVec; use intern::sym; -use crate::{Ident, Leaf, Punct, Spacing, Subtree, TokenTree, TokenTreesView}; +use crate::{Ident, Leaf, MAX_GLUED_PUNCT_LEN, Punct, Spacing, Subtree, TokenTree, TokenTreesView}; #[derive(Clone)] pub struct TtIter<'a, S> { @@ -111,7 +111,7 @@ impl<'a, S: Copy> TtIter<'a, S> { /// /// This method currently may return a single quotation, which is part of lifetime ident and /// conceptually not a punct in the context of mbe. Callers should handle this. - pub fn expect_glued_punct(&mut self) -> Result, 3>, ()> { + pub fn expect_glued_punct(&mut self) -> Result, MAX_GLUED_PUNCT_LEN>, ()> { let TtElement::Leaf(&Leaf::Punct(first)) = self.next().ok_or(())? else { return Err(()); }; @@ -145,7 +145,6 @@ impl<'a, S: Copy> TtIter<'a, S> { } ('-' | '!' | '*' | '/' | '&' | '%' | '^' | '+' | '<' | '=' | '>' | '|', '=', _) | ('-' | '=' | '>', '>', _) - | (_, _, Some(';')) | ('<', '-', _) | (':', ':', _) | ('.', '.', _) diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index 916e00b73bad..36ccb67f3b8d 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -22,6 +22,8 @@ use stdx::{impl_from, itertools::Itertools as _}; pub use text_size::{TextRange, TextSize}; +pub const MAX_GLUED_PUNCT_LEN: usize = 3; + #[derive(Clone, PartialEq, Debug)] pub struct Lit { pub kind: LitKind, @@ -243,6 +245,23 @@ impl TopSubtreeBuilder { self.token_trees.extend(tt.0.iter().cloned()); } + /// Like [`Self::extend_with_tt()`], but makes sure the new tokens will never be + /// joint with whatever comes after them. + pub fn extend_with_tt_alone(&mut self, tt: TokenTreesView<'_, S>) { + if let Some((last, before_last)) = tt.0.split_last() { + self.token_trees.reserve(tt.0.len()); + self.token_trees.extend(before_last.iter().cloned()); + let last = if let TokenTree::Leaf(Leaf::Punct(last)) = last { + let mut last = *last; + last.spacing = Spacing::Alone; + TokenTree::Leaf(Leaf::Punct(last)) + } else { + last.clone() + }; + self.token_trees.push(last); + } + } + pub fn expected_delimiters(&self) -> impl Iterator> { self.unclosed_subtree_indices.iter().rev().map(|&subtree_idx| { let TokenTree::Subtree(subtree) = &self.token_trees[subtree_idx] else {