Update fixes

derive/tests/grammar.pest

@@ -36,17 +36,17 @@ repeat_min_atomic = string{2..}
 repeat_max = string~{..2}
 repeat_max_atomic = string{..2}
 soi_at_start = pest::SOI - string
-repeat_mutate_stack = (pest::push('a'..'c') - ",")* - pest::pop - pest::pop - pest::pop
-repeat_mutate_stack_pop_all = (pest::push('a'..'c') - ",")* - pest::pop_all
+repeat_mutate_stack = (pest::stack::push('a'..'c') - ",")* - pest::stack::pop - pest::stack::pop - pest::stack::pop
+repeat_mutate_stack_pop_all = (pest::stack::push('a'..'c') - ",")* - pest::stack::pop_all
 will_fail = repeat_mutate_stack_pop_all - "FAIL"
 stack_resume_after_fail = will_fail | repeat_mutate_stack_pop_all
-peek_ = pest::push(range) - pest::push(range) - pest::peek - pest::peek
-peek_all = pest::push(range) - pest::push(range) - pest::peek_all
-peek_slice_23 = pest::push(range) - pest::push(range) - pest::push(range) ~ pest::push(range) ~ pest::push(range) ~ pest::peek[1..-2]
-pop_ = pest::push(range) - pest::push(range) - pest::pop - pest::pop
-pop_all = pest::push(range) - pest::push(range) - pest::pop_all
-pop_fail = pest::push(range) - !pest::pop - range - pest::pop
-checkpoint_restore = pest::push("") - (pest::push("a") - "b" - pest::pop | pest::drop - "b" | pest::pop - "a") - pest::EOI
+peek_ = pest::stack::push(range) - pest::stack::push(range) - pest::stack::peek - pest::stack::peek
+peek_all = pest::stack::push(range) - pest::stack::push(range) - pest::stack::peek_all
+peek_slice_23 = pest::stack::push(range) - pest::stack::push(range) - pest::stack::push(range) ~ pest::stack::push(range) ~ pest::stack::push(range) ~ pest::stack::peek[1..-2]
+pop_ = pest::stack::push(range) - pest::stack::push(range) - pest::stack::pop - pest::stack::pop
+pop_all = pest::stack::push(range) - pest::stack::push(range) - pest::stack::pop_all
+pop_fail = pest::stack::push(range) - !pest::stack::pop - range - pest::stack::pop
+checkpoint_restore = pest::stack::push("") - (pest::stack::push("a") - "b" - pest::stack::pop | pest::stack::drop - "b" | pest::stack::pop - "a") - pest::EOI
 
 /*
 ascii_digits = ASCII_DIGIT+

derive/tests/json.pest

@@ -0,0 +1,42 @@
+//! A parser for JSON file.
+//!
+//! And this is a example for JSON parser.
+//!
+//! Created by [Tomas Tauber](https://github.com/tomtau).
+//! Modified by [Boyi Huang](https://github.com/TheVeryDarkness).
+
+json = pest::SOI ~ value ~ pest::EOI
+
+/// Matches object, e.g.: `{ "foo": "bar" }`
+/// Foobar
+object = "{" ~ pair^* ~ "}"
+pair   = string ~ ":" ~ value
+
+array = "[" ~ value ~ ("," ~ value)* ~ "]" | "[" ~ "]"
+
+
+//////////////////////
+/// Matches value, e.g.: `"foo"`, `42`, `true`, `null`, `[]`, `{}`.
+//////////////////////
+value = string | number | object | array | bool | null
+
+string  = "\"" - inner - "\""
+inner   = (!("\"" | "\\") - pest::any)* - (escape ~ inner)?
+escape  = "\\" - ("\"" | "\\" | "/" | "b" | "f" | "n" | "r" | "t" | unicode)
+// FIXME: should be under pest::*
+ascii_hex_digit = ascii_digit | "A" | "B" | "C" | "D" | "E" | "F"
+ascii_digit = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
+ascii_nonzero_digit = "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
+unicode = "u" - ascii_hex_digit{4}
+
+number = "-"? - int - ("." ~ ascii_digit+ ~ exp? | exp)?
+int    = "0" | ascii_nonzero_digit - ascii_digit*
+exp    = ("E" | "e") - ("+" | "-")? - ascii_digit+
+
+bool = "true" | "false"
+
+null = "null"
+
+space = " " | "\t" | "\r" | "\n"
+~ = space*
+^ = space* - "," - space*

derive/tests/json.rs

@@ -0,0 +1,57 @@
+//! Created by [Tomas Tauber](https://github.com/tomtau).
+//! Modified by [Boyi Huang](https://github.com/TheVeryDarkness).
+
+/// Grammar rules of a sample JSON parser
+#[allow(missing_docs)]
+pub mod json {
+    use pest_derive::Parser;
+
+    /// JSON parser.
+    #[derive(Parser)]
+    #[grammar = "tests/json.pest"]
+    pub struct JsonParser;
+}
+
+#[cfg(test)]
+mod tests {
+    use pest::typed::{PairContainer, TypedParser};
+
+    use crate::{json, json::generics::*};
+    #[test]
+    fn json_basic_test() {
+        let sample1 = "{\"key\": \"value\"}";
+        let s1: json::rules::json = json::JsonParser::try_parse(sample1).unwrap();
+        let values = s1.vec_children_pairs();
+        assert_eq!(&values[0].rule, &json::Rule::value);
+        let obj = &values[0].children;
+        assert_eq!(obj[0].rule, json::Rule::object);
+        let pair = &obj[0].children;
+        assert_eq!(pair[0].rule, json::Rule::pair);
+        let key = &pair[0].children;
+        assert_eq!(key[0].rule, json::Rule::string);
+        assert_eq!(&sample1[key[0].start..key[0].end], "\"key\"");
+        let value = &key[1].children;
+        assert_eq!(value[0].rule, json::Rule::string);
+        assert_eq!(&sample1[value[0].start..value[0].end], "\"value\"");
+    }
+
+    /// Test in structural style.
+    #[test]
+    fn json_structural_test() {
+        let sample1 = "{\"key\": \"value\"}";
+        let s1: json::rules::json = json::JsonParser::try_parse(sample1).unwrap();
+        let value: json::rules::value<'_> = s1.content.field_1;
+        match *value.content {
+            Choice6::Choice2(object) => {
+                let (_, mut pairs, _) = object.content.into_tuple();
+                assert_eq!(pairs.content.len(), 1);
+                let pair = pairs.content.pop().unwrap();
+                let (key, _, value) = pair.content.into_tuple();
+                let (_, key, _) = key.content.into_tuple();
+                assert_eq!(key.span.as_str(), "key");
+                assert_eq!(value.span.as_str(), "\"value\"");
+            }
+            _ => {}
+        }
+    }
+}

derive/tests/sample.rs

@@ -0,0 +1,36 @@
+#![allow(unused_variables)]
+use anyhow::Result;
+use pest::typed::TypedNode as _;
+use pest_derive::Parser;
+
+#[derive(Parser)]
+#[grammar = "../meta/tests/pest3sample.pest"]
+struct Parser;
+
+#[test]
+fn comment() -> Result<()> {
+    rules::main::try_parse_partial("x x x /*x*/")?;
+    rules::main::try_parse("x x x /*x*/").unwrap_err();
+    Ok(())
+}
+
+#[test]
+fn skip_on_two_end() -> Result<()> {
+    rules::main::try_parse_partial("x x ")?;
+    rules::main::try_parse(" x x").unwrap_err();
+    rules::main::try_parse("x x ").unwrap_err();
+    Ok(())
+}
+
+#[test]
+fn post_skip() -> Result<()> {
+    let program = rules::program::try_parse("x x /*x*/")?;
+    Ok(())
+}
+
+#[test]
+fn pre_skip() -> Result<()> {
+    rules::program::try_parse("/* x x */ ")?;
+    rules::program::try_parse("/* x x */ x x")?;
+    Ok(())
+}

generator/Cargo.toml

@@ -9,7 +9,7 @@ proc-macro2 = "1.0"
 quote = "1.0"
 syn = "2.0"
 pest_meta.workspace = true
+pest.workspace = true
 
 [dev-dependencies]
 lazy_static = { version = "1.4.0" }
-pest.workspace = true

generator/src/common.rs

@@ -42,23 +42,17 @@ pub(crate) fn generate_rule_enum(rules: &[ParseRule], doc_comment: &DocComment)
         .map(|rule| {
             let rule_name = format_ident!("r#{}", rule.name);
 
-            match doc_comment.line_docs.get(&rule.name) {
-                Some(doc) => quote! {
+            let doc = &rule.doc;
+            quote! {
+                #(
                     #[doc = #doc]
-                    #rule_name,
-                },
-                None => quote! {
-                    #rule_name,
-                },
+                )*
+                #rule_name,
             }
         });
 
     let grammar_doc = &doc_comment.grammar_doc;
-    let grammar_doc = if let Some(grammar_doc) = grammar_doc {
-        quote! {#[doc = #grammar_doc]}
-    } else {
-        quote! {}
-    };
+    let grammar_doc = quote! {#(#[doc = #grammar_doc])*};
     quote! {
         #grammar_doc
         #[allow(dead_code, missing_docs, non_camel_case_types, clippy::upper_case_acronyms)]

generator/src/typed/attr.rs

@@ -21,6 +21,8 @@ pub(crate) fn parse_derive(ast: DeriveInput) -> (Ident, Generics, Vec<GrammarSou
             config.do_not_emit_span = get_bool(attr, "do_not_emit_span");
         } else if path.is_ident("no_warnings") {
             config.no_warnings = get_bool(attr, "no_warnings");
+        } else if path.is_ident("box_all_rules") {
+            config.box_all_rules = get_bool(attr, "box_all_rules");
         }
     }
 

generator/src/typed/config.rs

@@ -3,13 +3,15 @@ pub(crate) struct Config {
     pub emit_rule_reference: bool,
     pub do_not_emit_span: bool,
     pub no_warnings: bool,
+    pub box_all_rules: bool,
 }
 impl Default for Config {
     fn default() -> Self {
         Self {
             emit_rule_reference: false,
             do_not_emit_span: false,
             no_warnings: false,
+            box_all_rules: false,
         }
     }
 }