From 765116dcb4b9dc519f8da35a560aa21d416da18b Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Mon, 25 Sep 2023 20:39:13 +0200 Subject: [PATCH] Clean up macro parsing --- genco-macros/src/encoder.rs | 44 +++++----- genco-macros/src/lib.rs | 34 ++++++-- genco-macros/src/quote.rs | 40 ++++----- genco-macros/src/quote_fn.rs | 13 ++- genco-macros/src/quote_in.rs | 13 ++- genco-macros/src/static_buffer.rs | 13 +-- genco-macros/src/string_parser.rs | 135 ++++++++++++++++-------------- 7 files changed, 172 insertions(+), 120 deletions(-) diff --git a/genco-macros/src/encoder.rs b/genco-macros/src/encoder.rs index 7465744..94c8d6e 100644 --- a/genco-macros/src/encoder.rs +++ b/genco-macros/src/encoder.rs @@ -3,14 +3,15 @@ use crate::cursor::Cursor; use crate::fake::LineColumn; use crate::requirements::Requirements; use crate::static_buffer::StaticBuffer; +use crate::Ctxt; use proc_macro2::{Span, TokenStream}; use syn::Result; /// Struct to deal with emitting the necessary spacing. pub(crate) struct Encoder<'a> { - /// The identifier that received the input. - receiver: &'a syn::Ident, + /// Context for encoding. + cx: &'a Ctxt, /// Use to modify the initial line/column in case something was processed /// before the input was handed off to the quote parser. /// @@ -38,15 +39,15 @@ pub(crate) struct Encoder<'a> { impl<'a> Encoder<'a> { pub(crate) fn new( - receiver: &'a syn::Ident, + cx: &'a Ctxt, span_start: Option, span_end: Option, ) -> Self { Self { - receiver, + cx, span_start, span_end, - item_buffer: StaticBuffer::new(receiver), + item_buffer: StaticBuffer::new(cx), output: TokenStream::new(), last: None, last_start_column: None, @@ -151,29 +152,32 @@ impl<'a> Encoder<'a> { } pub(crate) fn encode_string(&mut self, has_eval: bool, stream: TokenStream) { + let Ctxt { receiver, module } = self.cx; + self.item_buffer.flush(&mut self.output); - let receiver = self.receiver; self.output.extend(q::quote! { - #receiver.append(genco::tokens::Item::OpenQuote(#has_eval)); + #receiver.append(#module::tokens::Item::OpenQuote(#has_eval)); #stream - #receiver.append(genco::tokens::Item::CloseQuote); + #receiver.append(#module::tokens::Item::CloseQuote); }); } pub(crate) fn encode_quoted(&mut self, s: syn::LitStr) { - let receiver = self.receiver; + let Ctxt { receiver, module } = self.cx; + self.item_buffer.flush(&mut self.output); self.output.extend(q::quote! { - #receiver.append(genco::tokens::Item::OpenQuote(false)); - #receiver.append(genco::tokens::ItemStr::Static(#s)); - #receiver.append(genco::tokens::Item::CloseQuote); + #receiver.append(#module::tokens::Item::OpenQuote(false)); + #receiver.append(#module::tokens::ItemStr::Static(#s)); + #receiver.append(#module::tokens::Item::CloseQuote); }); } pub(crate) fn encode_control(&mut self, control: Control) { - let receiver = self.receiver; + let Ctxt { receiver, .. } = self.cx; + self.item_buffer.flush(&mut self.output); match control.kind { @@ -193,7 +197,7 @@ impl<'a> Encoder<'a> { } pub(crate) fn encode_scope(&mut self, binding: Option, content: TokenStream) { - let receiver = self.receiver; + let Ctxt { receiver, .. } = self.cx; if binding.is_some() { self.item_buffer.flush(&mut self.output); @@ -209,7 +213,8 @@ impl<'a> Encoder<'a> { /// Encode an evaluation of the given expression. pub(crate) fn encode_eval_ident(&mut self, ident: syn::Ident) { - let receiver = self.receiver; + let Ctxt { receiver, .. } = self.cx; + self.item_buffer.flush(&mut self.output); self.output.extend(q::quote! { #receiver.append(#ident); @@ -218,7 +223,8 @@ impl<'a> Encoder<'a> { /// Encode an evaluation of the given expression. pub(crate) fn encode_eval(&mut self, expr: syn::Expr) { - let receiver = self.receiver; + let Ctxt { receiver, .. } = self.cx; + self.item_buffer.flush(&mut self.output); self.output.extend(q::quote! { #receiver.append(#expr); @@ -335,6 +341,8 @@ impl<'a> Encoder<'a> { /// Finalize the encoder. fn finalize(&mut self) -> Result<()> { + let Ctxt { receiver, .. } = self.cx; + // evaluate whitespace in case we have an explicit end span. while let Some(to) = self.span_end.take() { if let Some(from) = self.from() { @@ -345,8 +353,6 @@ impl<'a> Encoder<'a> { self.item_buffer.flush(&mut self.output); - let receiver = self.receiver; - while self.indents.pop().is_some() { self.output.extend(q::quote!(#receiver.unindent();)); } @@ -362,7 +368,7 @@ impl<'a> Encoder<'a> { to: LineColumn, to_span: Option, ) -> Result<()> { - let r = self.receiver; + let Ctxt { receiver: r, .. } = self.cx; // Do nothing if empty span. if from == to { diff --git a/genco-macros/src/lib.rs b/genco-macros/src/lib.rs index 7a9a68c..9679d84 100644 --- a/genco-macros/src/lib.rs +++ b/genco-macros/src/lib.rs @@ -11,6 +11,29 @@ extern crate proc_macro; use proc_macro2::Span; use syn::parse::{ParseStream, Parser as _}; +struct Ctxt { + receiver: syn::Ident, + module: syn::Path, +} + +impl Default for Ctxt { + fn default() -> Self { + let mut module = syn::Path { + leading_colon: None, + segments: syn::punctuated::Punctuated::default(), + }; + + module + .segments + .push(syn::Ident::new("genco", Span::call_site()).into()); + + Self { + receiver: syn::Ident::new("__genco_macros_toks", Span::call_site()), + module, + } + } +} + mod ast; mod cursor; mod encoder; @@ -659,9 +682,8 @@ mod token; /// [escape]: #escape-sequences #[proc_macro] pub fn quote(input: proc_macro::TokenStream) -> proc_macro::TokenStream { - let receiver = &syn::Ident::new("__genco_macros_toks", Span::call_site()); - - let parser = crate::quote::Quote::new(receiver); + let cx = Ctxt::default(); + let parser = crate::quote::Quote::new(&cx); let parser = move |stream: ParseStream| parser.parse(stream); @@ -670,10 +692,12 @@ pub fn quote(input: proc_macro::TokenStream) -> proc_macro::TokenStream { Err(e) => return proc_macro::TokenStream::from(e.to_compile_error()), }; - let check = req.into_check(receiver); + let check = req.into_check(&cx.receiver); + + let Ctxt { receiver, module } = &cx; let gen = q::quote! {{ - let mut #receiver = genco::tokens::Tokens::new(); + let mut #receiver = #module::tokens::Tokens::new(); { let mut #receiver = &mut #receiver; diff --git a/genco-macros/src/quote.rs b/genco-macros/src/quote.rs index 20b0a3f..0e97fdf 100644 --- a/genco-macros/src/quote.rs +++ b/genco-macros/src/quote.rs @@ -9,11 +9,11 @@ use crate::fake::Buf; use crate::fake::LineColumn; use crate::requirements::Requirements; use crate::string_parser::StringParser; +use crate::Ctxt; pub(crate) struct Quote<'a> { - /// Used to set the receiver identifier which is being modified by this - /// macro. - receiver: &'a syn::Ident, + /// Context variables. + cx: &'a Ctxt, /// Use to modify the initial line/column in case something was processed /// before the input was handed off to the quote parser. /// @@ -32,9 +32,9 @@ pub(crate) struct Quote<'a> { impl<'a> Quote<'a> { /// Construct a new quote parser. - pub(crate) fn new(receiver: &'a syn::Ident) -> Self { + pub(crate) fn new(cx: &'a Ctxt) -> Self { Self { - receiver, + cx, span_start: None, span_end: None, until_comma: false, @@ -43,9 +43,9 @@ impl<'a> Quote<'a> { } /// Construct a new quote parser that will only parse until the given token. - pub(crate) fn new_until_comma(receiver: &'a syn::Ident) -> Self { + pub(crate) fn new_until_comma(cx: &'a Ctxt) -> Self { Self { - receiver, + cx, span_start: None, span_end: None, until_comma: true, @@ -78,7 +78,7 @@ impl<'a> Quote<'a> { /// Parse until end of stream. pub(crate) fn parse(mut self, input: ParseStream) -> Result<(Requirements, TokenStream)> { - let mut encoder = Encoder::new(self.receiver, self.span_start, self.span_end); + let mut encoder = Encoder::new(self.cx, self.span_start, self.span_end); self.parse_inner(&mut encoder, input, 0)?; encoder.into_output() } @@ -90,7 +90,7 @@ impl<'a> Quote<'a> { if input.peek(Token![=>]) { input.parse::]>()?; - let (req, then_branch) = Quote::new(self.receiver).parse(input)?; + let (req, then_branch) = Quote::new(self.cx).parse(input)?; return Ok(( req, @@ -107,7 +107,7 @@ impl<'a> Quote<'a> { let content; syn::braced!(content in input); - let (r, then_branch) = Quote::new(self.receiver).parse(&content)?; + let (r, then_branch) = Quote::new(self.cx).parse(&content)?; req.merge_with(r); let else_branch = if input.peek(Token![else]) { @@ -116,7 +116,7 @@ impl<'a> Quote<'a> { let content; syn::braced!(content in input); - let (r, else_branch) = Quote::new(self.receiver).parse(&content)?; + let (r, else_branch) = Quote::new(self.cx).parse(&content)?; req.merge_with(r); Some(else_branch) @@ -151,9 +151,7 @@ impl<'a> Quote<'a> { let content; let paren = syn::parenthesized!(content in input); - let (r, join) = Quote::new(self.receiver) - .with_span(paren.span)? - .parse(&content)?; + let (r, join) = Quote::new(self.cx).with_span(paren.span)?.parse(&content)?; req.merge_with(r); Some(join) @@ -171,7 +169,7 @@ impl<'a> Quote<'a> { &content }; - let parser = Quote::new(self.receiver); + let parser = Quote::new(self.cx); let (r, stream) = parser.parse(input)?; req.merge_with(r); @@ -213,17 +211,15 @@ impl<'a> Quote<'a> { let block; syn::braced!(block in body); - let parser = Quote::new(self.receiver); + let parser = Quote::new(self.cx); parser.parse(&block)? } else if body.peek(token::Paren) { let block; let paren = syn::parenthesized!(block in body); - Quote::new(self.receiver) - .with_span(paren.span)? - .parse(&block)? + Quote::new(self.cx).with_span(paren.span)?.parse(&block)? } else { - let parser = Quote::new_until_comma(self.receiver); + let parser = Quote::new_until_comma(self.cx); parser.parse(&body)? }; @@ -348,7 +344,7 @@ impl<'a> Quote<'a> { )); } (LiteralName::Ident("str"), Some(content)) => { - let parser = StringParser::new(self.receiver, &self.buf, end)?; + let parser = StringParser::new(self.cx, &self.buf, end)?; let (options, r, stream) = parser.parse(&content)?; encoder.requirements.merge_with(r); @@ -358,7 +354,7 @@ impl<'a> Quote<'a> { encoder.encode( cursor, Ast::String { - has_eval: options.has_eval, + has_eval: options.has_eval.get(), stream, }, )?; diff --git a/genco-macros/src/quote_fn.rs b/genco-macros/src/quote_fn.rs index c7ea51d..0864cbc 100644 --- a/genco-macros/src/quote_fn.rs +++ b/genco-macros/src/quote_fn.rs @@ -2,20 +2,25 @@ use proc_macro2::TokenStream; use syn::parse::{Parse, ParseStream}; use syn::Result; +use crate::Ctxt; + pub(crate) struct QuoteFn { pub(crate) stream: TokenStream, } impl Parse for QuoteFn { fn parse(input: ParseStream) -> Result { - let receiver = &syn::Ident::new("__genco_macros_toks", input.span()); - let parser = crate::quote::Quote::new(receiver); + let cx = Ctxt::default(); + + let parser = crate::quote::Quote::new(&cx); let (req, output) = parser.parse(input)?; - let check = req.into_check(receiver); + let check = req.into_check(&cx.receiver); + + let Ctxt { receiver, module } = &cx; let stream = q::quote! { - genco::tokens::from_fn(move |#receiver| { + #module::tokens::from_fn(move |#receiver| { #output #check }) diff --git a/genco-macros/src/quote_in.rs b/genco-macros/src/quote_in.rs index 9ba201d..f047f49 100644 --- a/genco-macros/src/quote_in.rs +++ b/genco-macros/src/quote_in.rs @@ -3,6 +3,8 @@ use syn::parse::{Parse, ParseStream}; use syn::spanned::Spanned as _; use syn::{Result, Token}; +use crate::Ctxt; + pub(crate) struct QuoteIn { pub(crate) stream: TokenStream, } @@ -13,15 +15,18 @@ impl Parse for QuoteIn { let expr = input.parse::()?; input.parse::]>()?; - let receiver = &syn::Ident::new("__genco_macros_toks", expr.span()); - let parser = crate::quote::Quote::new(receiver); + let cx = Ctxt::default(); + + let parser = crate::quote::Quote::new(&cx); let (req, output) = parser.parse(input)?; - let check = req.into_check(receiver); + let check = req.into_check(&cx.receiver); + + let Ctxt { receiver, module } = &cx; // Give the assignment its own span to improve diagnostics. let assign_mut = q::quote_spanned! { expr.span() => - let #receiver: &mut genco::tokens::Tokens<_> = &mut #expr; + let #receiver: &mut #module::tokens::Tokens<_> = &mut #expr; }; let stream = q::quote! {{ diff --git a/genco-macros/src/static_buffer.rs b/genco-macros/src/static_buffer.rs index b6351d6..39dd4f6 100644 --- a/genco-macros/src/static_buffer.rs +++ b/genco-macros/src/static_buffer.rs @@ -1,16 +1,18 @@ use proc_macro2::{Span, TokenStream}; +use crate::Ctxt; + /// Buffer used to resolve static items. pub(crate) struct StaticBuffer<'a> { - receiver: &'a syn::Ident, + cx: &'a Ctxt, buffer: String, } impl<'a> StaticBuffer<'a> { /// Construct a new line buffer. - pub(crate) fn new(receiver: &'a syn::Ident) -> Self { + pub(crate) fn new(cx: &'a Ctxt) -> Self { Self { - receiver, + cx, buffer: String::new(), } } @@ -28,9 +30,10 @@ impl<'a> StaticBuffer<'a> { /// Flush the line buffer if necessary. pub(crate) fn flush(&mut self, tokens: &mut TokenStream) { if !self.buffer.is_empty() { - let receiver = self.receiver; + let Ctxt { receiver, module } = self.cx; + let s = syn::LitStr::new(&self.buffer, Span::call_site()); - tokens.extend(q::quote!(#receiver.append(genco::tokens::ItemStr::Static(#s));)); + tokens.extend(q::quote!(#receiver.append(#module::tokens::ItemStr::Static(#s));)); self.buffer.clear(); } } diff --git a/genco-macros/src/string_parser.rs b/genco-macros/src/string_parser.rs index 8e9cd6d..2376a33 100644 --- a/genco-macros/src/string_parser.rs +++ b/genco-macros/src/string_parser.rs @@ -1,9 +1,7 @@ //! Helper to parse quoted strings. -use crate::ast::LiteralName; -use crate::fake::{Buf, LineColumn}; -use crate::quote::parse_internal_function; -use crate::requirements::Requirements; +use std::cell::{Cell, RefCell}; +use std::fmt::Write; use proc_macro2::{Span, TokenStream, TokenTree}; use syn::parse::ParseStream; @@ -11,11 +9,17 @@ use syn::spanned::Spanned; use syn::token; use syn::Result; +use crate::ast::LiteralName; +use crate::fake::{Buf, LineColumn}; +use crate::quote::parse_internal_function; +use crate::requirements::Requirements; +use crate::Ctxt; + /// Options for the parsed string. -#[derive(Default, Clone, Copy)] +#[derive(Default)] pub(crate) struct Options { /// If the parsed string has any evaluation statements in it. - pub(crate) has_eval: bool, + pub(crate) has_eval: Cell, } fn adjust_start(start: LineColumn) -> LineColumn { @@ -33,150 +37,159 @@ fn adjust_end(end: LineColumn) -> LineColumn { } struct Encoder<'a> { - receiver: &'a syn::Ident, - cursor: Option, + cx: &'a Ctxt, span: Span, - count: usize, - buffer: String, - stream: TokenStream, + cursor: Cell>, + count: Cell, + buf: RefCell, + stream: RefCell, pub(crate) options: Options, } impl<'a> Encoder<'a> { - pub fn new(receiver: &'a syn::Ident, cursor: LineColumn, span: Span) -> Self { + pub fn new(cx: &'a Ctxt, cursor: LineColumn, span: Span) -> Self { Self { - receiver, - cursor: Some(cursor), + cx, span, - count: 0, - buffer: String::new(), - stream: TokenStream::new(), + cursor: Cell::new(Some(cursor)), + count: Cell::new(0), + buf: RefCell::new(String::new()), + stream: RefCell::new(TokenStream::new()), options: Options::default(), } } - pub(crate) fn finalize(mut self, end: LineColumn) -> Result<(Options, TokenStream)> { + pub(crate) fn finalize(self, end: LineColumn) -> Result<(Options, TokenStream)> { self.flush(Some(end), None)?; - Ok((self.options, self.stream)) + Ok((self.options, self.stream.into_inner())) } /// Encode a single character and replace the cursor with the given /// location. - pub(crate) fn encode_char(&mut self, c: char, from: LineColumn, to: LineColumn) -> Result<()> { + pub(crate) fn encode_char(&self, c: char, from: LineColumn, to: LineColumn) -> Result<()> { self.flush_whitespace(Some(from), Some(to))?; - self.buffer.push(c); - self.cursor = Some(to); + self.buf.borrow_mut().push(c); + self.cursor.set(Some(to)); Ok(()) } /// Encode a string directly to the static buffer as an optimization. pub(crate) fn encode_str( - &mut self, + &self, s: &str, from: LineColumn, to: Option, ) -> Result<()> { self.flush_whitespace(Some(from), to)?; - self.buffer.push_str(s); + self.buf.borrow_mut().push_str(s); Ok(()) } /// Eval the given identifier. pub(crate) fn eval_ident( - &mut self, + &self, ident: &syn::Ident, from: LineColumn, to: Option, ) -> Result<()> { + let Ctxt { receiver, module } = self.cx; + self.flush(Some(from), to)?; - let receiver = self.receiver; let ident = syn::LitStr::new(&ident.to_string(), ident.span()); - self.stream.extend(q::quote! { - #receiver.append(genco::tokens::Item::OpenEval); - #receiver.append(genco::tokens::Item::Literal(genco::tokens::ItemStr::Static(#ident))); - #receiver.append(genco::tokens::Item::CloseEval); + self.stream.borrow_mut().extend(q::quote! { + #receiver.append(#module::tokens::Item::OpenEval); + #receiver.append(#module::tokens::Item::Literal(#module::tokens::ItemStr::Static(#ident))); + #receiver.append(#module::tokens::Item::CloseEval); }); - self.options.has_eval = true; + self.options.has_eval.set(true); Ok(()) } /// Eval the given expression. pub(crate) fn eval_stream( - &mut self, + &self, expr: TokenStream, from: LineColumn, to: Option, ) -> Result<()> { self.flush(Some(from), to)?; - let receiver = self.receiver; - self.stream.extend(q::quote! { - #receiver.append(genco::tokens::Item::OpenEval); + let Ctxt { receiver, module } = self.cx; + + self.stream.borrow_mut().extend(q::quote! { + #receiver.append(#module::tokens::Item::OpenEval); #expr - #receiver.append(genco::tokens::Item::CloseEval); + #receiver.append(#module::tokens::Item::CloseEval); }); - self.options.has_eval = true; + self.options.has_eval.set(true); Ok(()) } /// Extend the content of the string with the given raw stream. pub(crate) fn raw_expr( - &mut self, + &self, expr: &syn::Expr, from: LineColumn, to: Option, ) -> Result<()> { self.flush(Some(from), to)?; - let receiver = self.receiver; - self.stream.extend(q::quote! { + + let Ctxt { receiver, .. } = self.cx; + + self.stream.borrow_mut().extend(q::quote! { #receiver.append(#expr); }); Ok(()) } pub(crate) fn extend_tt( - &mut self, + &self, tt: &TokenTree, from: LineColumn, to: Option, ) -> Result<()> { self.flush_whitespace(Some(from), to)?; - self.buffer.push_str(&tt.to_string()); + write!(self.buf.borrow_mut(), "{tt}").unwrap(); Ok(()) } /// Flush the outgoing buffer. - pub fn flush(&mut self, from: Option, to: Option) -> Result<()> { + pub fn flush(&self, from: Option, to: Option) -> Result<()> { + let Ctxt { receiver, module } = self.cx; + self.flush_whitespace(from, to)?; - let receiver = self.receiver; - if self.buffer.is_empty() { - return Ok(()); - } + let lit = { + let buf = self.buf.borrow(); + + if buf.is_empty() { + return Ok(()); + } - let lit = syn::LitStr::new(&self.buffer, self.span); + syn::LitStr::new(buf.as_str(), self.span) + }; - self.count += 1; + self.count.set(self.count.get().wrapping_add(1)); - self.stream.extend(q::quote! { - #receiver.append(genco::tokens::ItemStr::Static(#lit)); + self.stream.borrow_mut().extend(q::quote! { + #receiver.append(#module::tokens::ItemStr::Static(#lit)); }); - self.buffer.clear(); + self.buf.borrow_mut().clear(); Ok(()) } /// Flush the outgoing buffer. pub(crate) fn flush_whitespace( - &mut self, + &self, from: Option, to: Option, ) -> Result<()> { - if let (Some(from), Some(cursor)) = (from, self.cursor) { + if let (Some(from), Some(cursor)) = (from, self.cursor.get()) { if cursor.line != from.line { return Err(syn::Error::new( self.span, @@ -185,17 +198,17 @@ impl<'a> Encoder<'a> { } for _ in 0..from.column.saturating_sub(cursor.column) { - self.buffer.push(' '); + self.buf.borrow_mut().push(' '); } } - self.cursor = to; + self.cursor.set(to); Ok(()) } } pub struct StringParser<'a> { - receiver: &'a syn::Ident, + cx: &'a Ctxt, buf: &'a Buf, start: LineColumn, end: LineColumn, @@ -203,11 +216,11 @@ pub struct StringParser<'a> { } impl<'a> StringParser<'a> { - pub(crate) fn new(receiver: &'a syn::Ident, buf: &'a Buf, span: Span) -> syn::Result { + pub(crate) fn new(cx: &'a Ctxt, buf: &'a Buf, span: Span) -> syn::Result { let cursor = buf.cursor(span)?; Ok(Self { - receiver, + cx, buf, // Note: adjusting span since we expect the quoted string to be // withing a block, where the interior span is one character pulled @@ -220,7 +233,7 @@ impl<'a> StringParser<'a> { pub(crate) fn parse(self, input: ParseStream) -> Result<(Options, Requirements, TokenStream)> { let mut requirements = Requirements::default(); - let mut encoder = Encoder::new(self.receiver, self.start, self.span); + let encoder = Encoder::new(self.cx, self.start, self.span); while !input.is_empty() { if input.peek(syn::Token![$]) && input.peek2(syn::Token![$]) { @@ -274,7 +287,7 @@ impl<'a> StringParser<'a> { let content; let end = syn::parenthesized!(content in input).span; - let (req, stream) = crate::quote::Quote::new(self.receiver) + let (req, stream) = crate::quote::Quote::new(self.cx) .with_span(content.span())? .parse(&content)?; requirements.merge_with(req);