proc_macro_utils/
lib.rs

1//! Some useful functions on [`proc_macro`] and [`proc_macro2`] types
2//!
3//! E.g. [pushing tokens onto `TokenStream`](TokenStreamExt::push) and [testing
4//! for specific punctuation on `TokenTree` and `Punct`](TokenTreePunct)
5//!
6//! It also adds the [`assert_tokens!`] and [`assert_expansion!`] macros to
7//! improve unit testability for `proc-macros`.
8#![warn(clippy::pedantic, missing_docs)]
9#![cfg_attr(docsrs, feature(doc_auto_cfg))]
10#![deny(rustdoc::all)]
11
12#[cfg(all(doc, feature = "proc-macro2"))]
13use proc_macro2::{Punct, Spacing};
14
15#[cfg(feature = "proc-macro")]
16extern crate proc_macro;
17
18/// Parsing of simple rust structures without syn
19#[cfg(feature = "parser")]
20mod parser;
21#[cfg(feature = "parser")]
22pub use parser::TokenParser;
23
24#[cfg(feature = "parser")]
25#[macro_use]
26mod assert;
27
28#[cfg(feature = "parser")]
29#[doc(hidden)]
30pub mod __private;
31
32mod sealed {
33    pub trait Sealed {}
34
35    macro_rules! sealed {
36        [$($ty:ident),* $(,)?] => {$(
37            #[cfg(feature = "proc-macro")]
38            impl Sealed for proc_macro::$ty {}
39            #[cfg(feature = "proc-macro2")]
40            impl Sealed for proc_macro2::$ty {}
41        )*};
42    }
43
44    sealed![TokenStream, TokenTree, Punct, Literal, Group];
45}
46
47macro_rules! once {
48    (($($tts:tt)*) $($tail:tt)*) => {
49        $($tts)*
50    };
51}
52
53macro_rules! attr {
54    (($($attr:tt)*), $($item:tt)+) => {
55        $(#$attr)* $($item)+
56    };
57}
58
59macro_rules! trait_def {
60    ($item_attr:tt, $trait:ident, $($fn_attr:tt, $fn:ident, $({$($gen:tt)*})?, $args:tt, $($ret:ty)?),*) => {
61        attr!($item_attr,
62        pub trait $trait: crate::sealed::Sealed {
63            $(attr!($fn_attr, fn $fn $($($gen)*)? $args $(-> $ret)?;);)*
64        });
65    };
66}
67
68macro_rules! trait_impl {
69    ($trait:ident, $type:ident, $($fn_attr:tt, $fn:ident, $({$($gen:tt)*})?, $args:tt, $($ret:ty)?, $stmts:tt),*) => {
70        impl $trait for $type {
71            $(attr!($fn_attr, fn $fn $($($gen)*)? $args $(-> $ret)? $stmts);)*
72        }
73    };
74}
75
76macro_rules! impl_via_trait {
77    ($(
78        $(#$trait_attr:tt)*
79        impl $trait:ident for $type:ident {
80            $($(#$fn_attr:tt)*
81            fn $fn:ident $({$($gen:tt)*})? ($($args:tt)*)  $(-> $ret:ty)? { $($stmts:tt)* })*
82        }
83    )+) => {
84        once!($((trait_def!(($($trait_attr)*), $trait, $(($($fn_attr)*), $fn,$({$($gen)*})?, ($($args)*), $($ret)?),*);))+);
85        #[cfg(feature = "proc-macro")]
86        const _: () = {
87            use proc_macro::*;
88            $(trait_impl!($trait, $type, $(($($fn_attr)*), $fn, $({$($gen)*})?, ($($args)*), $($ret)?, {$($stmts)*}),*);)+
89        };
90        #[cfg(feature = "proc-macro2")]
91        const _:() = {
92            use proc_macro2::*;
93            $(trait_impl!($trait, $type, $(($($fn_attr)*), $fn, $({$($gen)*})?, ($($args)*), $($ret)?, {$($stmts)*}),*);)+
94        };
95    };
96    (
97        mod $mod:ident, $mod2:ident {
98            $(
99                $(#$trait_attr:tt)*
100                impl $trait:ident$($doc:literal)?, $trait2:ident$($doc2:literal)?  for $type:ident {
101                    $($(#$fn_attr:tt)*
102                    fn $fn:ident $({$($gen:tt)*})? ($($args:tt)*) $(-> $ret:ty)? { $($stmts:tt)* })*
103                }
104            )+
105        }
106    ) => {
107        #[cfg(feature = "proc-macro")]
108        once!(($(pub use $mod::$trait;)+));
109        #[cfg(feature = "proc-macro")]
110        mod $mod {
111            use proc_macro::*;
112            once!($((trait_def!(($($trait_attr)* $([doc=$doc])?), $trait, $(($($fn_attr)*), $fn, $({$($gen)*})?, ($($args)*), $($ret)?),*);))+);
113            $(trait_impl!($trait, $type, $(($($fn_attr)*), $fn, $({$($gen)*})?, ($($args)*), $($ret)?, {$($stmts)*}),*);)+
114        }
115        #[cfg(feature = "proc-macro2")]
116        once!(($(pub use $mod2::$trait2;)+));
117        #[cfg(feature = "proc-macro2")]
118        mod $mod2 {
119            use proc_macro2::*;
120            once!($((trait_def!(($($trait_attr)*$([doc=$doc2])?), $trait2, $(($($fn_attr)*), $fn, $({$($gen)*})?, ($($args)*), $($ret)?),*);))+);
121            $(trait_impl!($trait2, $type, $(($($fn_attr)*), $fn, $({$($gen)*})?, ($($args)*), $($ret)?, {$($stmts)*}),*);)+
122        }
123    };
124}
125
126impl_via_trait! {
127    mod token_stream_ext, token_stream2_ext {
128        /// Generic extensions for
129        impl TokenStreamExt "[`proc_macro::TokenStream`]", TokenStream2Ext "[`proc_macro2::TokenStream`]" for TokenStream {
130            /// Pushes a single [`TokenTree`] onto the token stream.
131            fn push(&mut self, token: TokenTree) {
132                self.extend(std::iter::once(token))
133            }
134            /// Creates a [`TokenParser`](crate::TokenParser) from this token stream.
135            #[cfg(feature = "parser")]
136            fn parser(self) -> crate::TokenParser<proc_macro2::token_stream::IntoIter> {
137                #[allow(clippy::useless_conversion)]
138                proc_macro2::TokenStream::from(self).into()
139            }
140
141            /// Creates a [`TokenParser`](crate::TokenParser) from this token stream.
142            ///
143            /// Allows to specify the length of the [peeker buffer](crate::TokenParser#peeking).
144            #[cfg(feature = "parser")]
145            fn parser_generic{<const PEEKER_LEN: usize>}(self) -> crate::TokenParser<proc_macro2::token_stream::IntoIter, PEEKER_LEN> {
146                #[allow(clippy::useless_conversion)]
147                proc_macro2::TokenStream::from(self).into()
148            }
149        }
150    }
151}
152
153macro_rules! token_tree_ext {
154    ($($a:literal, $token:literal, $is:ident, $as:ident, $into:ident, $variant:ident);+$(;)?) => {
155        impl_via_trait! {
156            mod token_tree_ext, token_tree2_ext {
157                /// Generic extensions for
158                impl TokenTreeExt "[`proc_macro::TokenTree`]", TokenTree2Ext "[`proc_macro2::TokenTree`]"  for TokenTree {
159                    $(
160                        #[doc = concat!("Tests if the token tree is ", $a, " ", $token, ".")]
161                        #[must_use]
162                        fn $is(&self) -> bool {
163                            matches!(self, Self::$variant(_))
164                        }
165                        #[doc = concat!("Get the [`", stringify!($variant), "`] inside this token tree, or [`None`] if it isn't ", $a, " ", $token, ".")]
166                        #[must_use]
167                        fn $as(&self) -> Option<&$variant> {
168                            if let Self::$variant(inner) = &self {
169                                Some(inner)
170                            } else {
171                                None
172                            }
173                        }
174                        #[doc = concat!("Get the [`", stringify!($variant), "`] inside this token tree, or [`None`] if it isn't ", $a, " ", $token, ".")]
175                        #[must_use]
176                        fn $into(self) -> Option<$variant> {
177                            if let Self::$variant(inner) = self {
178                                Some(inner)
179                            } else {
180                                None
181                            }
182                        }
183                    )*
184                }
185            }
186        }
187    };
188}
189
190token_tree_ext!(
191    "a", "group", is_group, group, into_group, Group;
192    "an", "ident", is_ident, ident, into_ident, Ident;
193    "a", "punctuation", is_punct, punct, into_punct, Punct;
194    "a", "literal", is_literal, literal, into_literal, Literal;
195);
196
197macro_rules! punctuations {
198    ($($char:literal as $name:ident),*) => {
199        impl_via_trait!{
200            /// Trait to test for punctuation
201            impl TokenTreePunct for TokenTree {
202                $(#[doc = concat!("Tests if the token is `", $char, "`")]
203                #[must_use]
204                fn $name(&self) -> bool {
205                    matches!(self, TokenTree::Punct(punct) if punct.$name())
206                })*
207                /// Tests if token is followed by some none punctuation token or whitespace.
208                #[must_use]
209                fn is_alone(&self) -> bool {
210                    matches!(self, TokenTree::Punct(punct) if punct.is_alone())
211                }
212                /// Tests if token is followed by another punct and can potentially be combined into
213                /// a multi-character operator.
214                #[must_use]
215                fn is_joint(&self) -> bool {
216                    matches!(self, TokenTree::Punct(punct) if punct.is_joint())
217                }
218                /// If sets the [`spacing`](Punct::spacing) of a punct to [`Alone`](Spacing::Alone).
219                #[must_use]
220                fn alone(self) -> Self {
221                    match self {
222                        Self::Punct(p) => Self::Punct(p.alone()),
223                        it => it
224                    }
225                }
226            }
227            impl TokenTreePunct for Punct {
228                $(fn $name(&self) -> bool {
229                    self.as_char() == $char
230                })*
231                fn is_alone(&self) -> bool {
232                    self.spacing() == Spacing::Alone
233                }
234                fn is_joint(&self) -> bool {
235                    self.spacing() == Spacing::Joint
236                }
237                fn alone(self) -> Self {
238                    if self.is_alone() {
239                        self
240                    } else {
241                        let mut this = Punct::new(self.as_char(), Spacing::Alone);
242                        this.set_span(self.span());
243                        this
244                    }
245                }
246            }
247        }
248    };
249}
250
251punctuations![
252    '=' as is_equals,
253    '<' as is_less_than,
254    '>' as is_greater_than,
255    '!' as is_exclamation,
256    '~' as is_tilde,
257    '+' as is_plus,
258    '-' as is_minus,
259    '*' as is_asterix, // TODO naming
260    '/' as is_slash,
261    '%' as is_percent,
262    '^' as is_caret,
263    '&' as is_and,
264    '|' as is_pipe,
265    '@' as is_at,
266    '.' as is_dot,
267    ',' as is_comma,
268    ';' as is_semi,
269    ':' as is_colon,
270    '#' as is_pound,
271    '$' as is_dollar,
272    '?' as is_question,
273    '\'' as is_quote // TODO naming
274];
275
276macro_rules! delimited {
277    ($($delimiter:ident as $name:ident : $doc:literal),*) => {
278        impl_via_trait!{
279            /// Trait to test for delimiters of groups
280            impl Delimited for TokenTree {
281                $(#[doc = concat!("Tests if the token is a group with ", $doc)]
282                #[must_use]
283                fn $name(&self) -> bool {
284                    matches!(self, TokenTree::Group(group) if group.$name())
285                })*
286            }
287            impl Delimited for Group {
288                $(#[doc = concat!("Tests if a group has ", $doc)]
289                #[must_use]
290                fn $name(&self) -> bool {
291                    matches!(self.delimiter(), Delimiter::$delimiter)
292                })*
293            }
294        }
295    };
296}
297
298delimited![
299    Parenthesis as is_parenthesized: " parentheses (`( ... )`)",
300    Brace as is_braced: " braces (`{ ... }`)",
301    Bracket as is_bracketed: " brackets (`[ ... ]`)",
302    None as is_implicitly_delimited: " no delimiters (`Ø ... Ø`)"
303];
304
305impl_via_trait! {
306    /// Trait to parse literals
307    impl TokenTreeLiteral for TokenTree {
308        /// Tests if the token is a string literal.
309        #[must_use]
310        fn is_string(&self) -> bool {
311            self.literal().is_some_and(TokenTreeLiteral::is_string)
312        }
313
314        /// Returns the string contents if it is a string literal.
315        #[must_use]
316        fn string(&self) -> Option<String> {
317            self.literal().and_then(TokenTreeLiteral::string)
318        }
319    }
320
321    impl TokenTreeLiteral for Literal {
322        fn is_string(&self) -> bool {
323            let s = self.to_string();
324            s.starts_with('"') || s.starts_with("r\"") || s.starts_with("r#")
325        }
326        fn string(&self) -> Option<String> {
327            let lit = self.to_string();
328            if lit.starts_with('"') {
329                Some(resolve_escapes(&lit[1..lit.len() - 1]))
330            } else if lit.starts_with('r') {
331                let pounds = lit.chars().skip(1).take_while(|&c| c == '#').count();
332                Some(lit[2 + pounds..lit.len() - pounds - 1].to_owned())
333            } else {
334                None
335            }
336        }
337    }
338}
339
340// Implemented following https://doc.rust-lang.org/reference/tokens.html#string-literals
341// #[allow(clippy::needless_continue)]
342fn resolve_escapes(mut s: &str) -> String {
343    let mut out = String::new();
344    while !s.is_empty() {
345        if s.starts_with('\\') {
346            match s.as_bytes()[1] {
347                b'x' => {
348                    out.push(
349                        char::from_u32(u32::from_str_radix(&s[2..=3], 16).expect("valid escape"))
350                            .expect("valid escape"),
351                    );
352                    s = &s[4..];
353                }
354                b'u' => {
355                    let len = s[3..].find('}').expect("valid escape");
356                    out.push(
357                        char::from_u32(u32::from_str_radix(&s[3..len], 16).expect("valid escape"))
358                            .expect("valid escape"),
359                    );
360                    s = &s[3 + len..];
361                }
362                b'n' => {
363                    out.push('\n');
364                    s = &s[2..];
365                }
366                b'r' => {
367                    out.push('\r');
368                    s = &s[2..];
369                }
370                b't' => {
371                    out.push('\t');
372                    s = &s[2..];
373                }
374                b'\\' => {
375                    out.push('\\');
376                    s = &s[2..];
377                }
378                b'0' => {
379                    out.push('\0');
380                    s = &s[2..];
381                }
382                b'\'' => {
383                    out.push('\'');
384                    s = &s[2..];
385                }
386                b'"' => {
387                    out.push('"');
388                    s = &s[2..];
389                }
390                b'\n' => {
391                    s = &s[..s[2..]
392                        .find(|c: char| !c.is_ascii_whitespace())
393                        .unwrap_or(s.len())];
394                }
395                c => unreachable!(
396                    "TokenStream string literals should only contain valid escapes, found `\\{c}`"
397                ),
398            }
399        } else {
400            let len = s.find('\\').unwrap_or(s.len());
401            out.push_str(&s[..len]);
402            s = &s[len..];
403        }
404    }
405    out
406}
407
408#[cfg(all(test, feature = "proc-macro2"))]
409mod test {
410    use proc_macro2::{Punct, Spacing, TokenTree};
411    use quote::quote;
412
413    use super::*;
414
415    #[test]
416    fn punctuation() {
417        let mut tokens = quote! {=<>!$~+-*/%^|@.,;:#$?'a}.into_iter();
418        assert!(tokens.next().unwrap().is_equals());
419        assert!(tokens.next().unwrap().is_less_than());
420        assert!(tokens.next().unwrap().is_greater_than());
421        assert!(tokens.next().unwrap().is_exclamation());
422        assert!(tokens.next().unwrap().is_dollar());
423        assert!(tokens.next().unwrap().is_tilde());
424        assert!(tokens.next().unwrap().is_plus());
425        assert!(tokens.next().unwrap().is_minus());
426        assert!(tokens.next().unwrap().is_asterix());
427        assert!(tokens.next().unwrap().is_slash());
428        assert!(tokens.next().unwrap().is_percent());
429        assert!(tokens.next().unwrap().is_caret());
430        assert!(tokens.next().unwrap().is_pipe());
431        assert!(tokens.next().unwrap().is_at());
432        assert!(tokens.next().unwrap().is_dot());
433        assert!(tokens.next().unwrap().is_comma());
434        assert!(tokens.next().unwrap().is_semi());
435        assert!(tokens.next().unwrap().is_colon());
436        assert!(tokens.next().unwrap().is_pound());
437        assert!(tokens.next().unwrap().is_dollar());
438        assert!(tokens.next().unwrap().is_question());
439        assert!(tokens.next().unwrap().is_quote());
440    }
441
442    #[test]
443    fn token_stream_ext() {
444        let mut tokens = quote!(a);
445        tokens.push(TokenTree::Punct(Punct::new(',', Spacing::Alone)));
446        assert_eq!(tokens.to_string(), "a ,");
447    }
448
449    #[test]
450    fn token_tree_ext() {
451        let mut tokens = quote!({group} ident + "literal").into_iter().peekable();
452        assert!(tokens.peek().unwrap().is_group());
453        assert!(matches!(
454            tokens.next().unwrap().group().unwrap().to_string().as_str(),
455            "{ group }" | "{group}"
456        ));
457        assert!(tokens.peek().unwrap().is_ident());
458        assert_eq!(tokens.next().unwrap().ident().unwrap().to_string(), "ident");
459        assert!(tokens.peek().unwrap().is_punct());
460        assert_eq!(tokens.next().unwrap().punct().unwrap().to_string(), "+");
461        assert!(tokens.peek().unwrap().is_literal());
462        assert_eq!(
463            tokens.next().unwrap().literal().unwrap().to_string(),
464            "\"literal\""
465        );
466    }
467
468    #[test]
469    fn test() {}
470}
proc_macro_utils/lib.rs

proc_macro_utils/
lib.rs
