use separated Vec for each rule_type instead of TokenData

rusty_lr_core/src/nonterminal.rs

@@ -51,52 +51,3 @@ pub enum NonTerminalType {
     /// "abc" or b"abc"
     LiteralString,
 }
-
-/// A trait for token that holds data.
-/// This will be used for data stack in the parser.
-pub trait TokenData: Sized {
-    /// Type for terminal symbols
-    type Term;
-    /// Type for non-terminal symbols - this must be enum type that was auto-generated by rusty_lr
-    type NonTerm;
-    /// Type for user data that is passed to the parser from the user.
-    type UserData;
-    /// Type for `Err` variant returned by reduce action
-    type ReduceActionError;
-    /// The value of the start symbol
-    type StartType;
-    /// Type for location of the token
-    type Location: crate::Location;
-
-    /// performs a reduce action with the given rule index
-    fn reduce_action(
-        // the child tokens for the reduction
-        // the caller (usually from generated code) must pops all of the tokens used for this reduce_action
-        data_stack: &mut Vec<Self>,
-        location_stack: &mut Vec<Self::Location>,
-
-        // the index of the production rule to reduce
-        rule_index: usize,
-
-        // for runtime-conflict-resolve.
-        // if this variable is set to false in the action, the shift action will not be performed. (GLR parser)
-        shift: &mut bool,
-        // the lookahead token that caused this reduce action
-        lookahead: &crate::TerminalSymbol<Self::Term>,
-        // user input data
-        userdata: &mut Self::UserData,
-        // location of this non-terminal, e.g. `@$`
-        location0: &mut Self::Location,
-    ) -> Result<Self, Self::ReduceActionError>;
-
-    /// create new token data for error recovery
-    fn new_error() -> Self;
-
-    /// create new token data that is empty.
-    fn new_empty() -> Self {
-        Self::new_error()
-    }
-
-    /// create new terminal variant
-    fn new_terminal(term: Self::Term) -> Self;
-}

rusty_lr_core/src/parser/data_stack.rs

@@ -0,0 +1,58 @@
+/// A trait for data stack in the parser.
+///
+/// Since each non-terminal could have different ruletypes,
+/// this effectively handles those rule types into separated `Vec` stack,
+/// instead of using enum of rule types (since it would be costful at memory aspects if the size differs significantly).
+/// For people who is curious about the implementation details,
+/// you should see the actual generated `DataStack` structs, like `GrammarDataStack` in `rusty_lr_parser/src/parser/parser_expanded.rs`.
+pub trait DataStack: Sized + Default {
+    /// Type for terminal symbols
+    type Term;
+    /// Type for non-terminal symbols - this must be enum type that was auto-generated by rusty_lr
+    type NonTerm;
+    /// Type for user data that is passed to the parser from the user.
+    type UserData;
+    /// Type for `Err` variant returned by reduce action
+    type ReduceActionError;
+    /// The value of the start symbol
+    type StartType;
+    /// Type for location of the token
+    type Location: crate::Location;
+
+    fn pop_start(&mut self) -> Option<Self::StartType>;
+    fn pop(&mut self);
+    fn push_terminal(&mut self, term: Self::Term);
+    fn push_empty(&mut self);
+
+    fn clear(&mut self);
+    fn reserve(&mut self, additional: usize);
+    fn with_capacity(capacity: usize) -> Self {
+        let mut self_: Self = Default::default();
+        self_.reserve(capacity);
+        self_
+    }
+
+    fn split_off(&mut self, at: usize) -> Self;
+    fn append(&mut self, other: &mut Self);
+
+    /// performs a reduce action with the given rule index
+    fn reduce_action(
+        // the child tokens for the reduction
+        // the caller (usually from generated code) must pops all of the tokens used for this reduce_action
+        data_stack: &mut Self,
+        location_stack: &mut Vec<Self::Location>,
+
+        // the index of the production rule to reduce
+        rule_index: usize,
+
+        // for runtime-conflict-resolve.
+        // if this variable is set to false in the action, the shift action will not be performed. (GLR parser)
+        shift: &mut bool,
+        // the lookahead token that caused this reduce action
+        lookahead: &crate::TerminalSymbol<Self::Term>,
+        // user input data
+        userdata: &mut Self::UserData,
+        // location of this non-terminal, e.g. `@$`
+        location0: &mut Self::Location,
+    ) -> Result<(), Self::ReduceActionError>;
+}

rusty_lr_core/src/parser/deterministic/context.rs

@@ -4,33 +4,33 @@ use std::hash::Hash;
 use super::ParseError;
 
 use crate::nonterminal::NonTerminal;
-use crate::nonterminal::TokenData;
+use crate::parser::data_stack::DataStack;
 use crate::parser::state::Index;
 use crate::parser::Parser;
 use crate::parser::Precedence;
 use crate::parser::State;
 use crate::TerminalSymbol;
 
 /// A struct that maintains the current state and the values associated with each symbol.
-pub struct Context<Data: TokenData, StateIndex> {
+pub struct Context<Data: DataStack, StateIndex> {
     /// stacks hold the values associated with each shifted symbol.
     pub state_stack: Vec<StateIndex>,
-    pub(crate) data_stack: Vec<Data>,
+    pub(crate) data_stack: Data,
     pub(crate) location_stack: Vec<Data::Location>,
     pub(crate) precedence_stack: Vec<Precedence>,
     /// Tree stack for tree representation of the parse.
     #[cfg(feature = "tree")]
     pub(crate) tree_stack: crate::tree::TreeList<Data::Term, Data::NonTerm>,
 }
 
-impl<Data: TokenData, StateIndex: Index + Copy> Context<Data, StateIndex> {
+impl<Data: DataStack, StateIndex: Index + Copy> Context<Data, StateIndex> {
     /// Create a new context.
     /// `state_stack` is initialized with [0] (root state).
     pub fn new() -> Self {
         Context {
             state_stack: vec![StateIndex::from_usize_unchecked(0)],
 
-            data_stack: Vec::new(),
+            data_stack: Default::default(),
             location_stack: Vec::new(),
             precedence_stack: Vec::new(),
 
@@ -48,7 +48,7 @@ impl<Data: TokenData, StateIndex: Index + Copy> Context<Data, StateIndex> {
         Context {
             state_stack,
 
-            data_stack: Vec::with_capacity(capacity),
+            data_stack: Data::with_capacity(capacity),
             location_stack: Vec::with_capacity(capacity),
             precedence_stack: Vec::with_capacity(capacity),
 
@@ -63,19 +63,14 @@ impl<Data: TokenData, StateIndex: Index + Copy> Context<Data, StateIndex> {
         userdata: &mut Data::UserData,
     ) -> Result<Data::StartType, ParseError<Data>>
     where
-        Data: TryInto<Data::StartType>,
         Data::Term: Clone,
         Data::NonTerm: Hash + Eq + Copy + NonTerminal,
     {
         self.feed_eof(parser, userdata)?;
 
         // data_stack must be <Start> <EOF> in this point
-        self.data_stack.pop();
-        if let Ok(start) = self.data_stack.pop().unwrap().try_into() {
-            Ok(start)
-        } else {
-            unreachable!("data stack must have start symbol at this point");
-        }
+        // Since <EOF> does not have ruletype, no need to pop
+        Ok(self.data_stack.pop_start().unwrap())
     }
 
     /// Check if current context can be terminated and get the value of the start symbol.
@@ -289,8 +284,7 @@ impl<Data: TokenData, StateIndex: Index + Copy> Context<Data, StateIndex> {
                         .push(crate::tree::Tree::new_terminal(term.clone()));
 
                     // shift with `error` token
-                    self.data_stack
-                        .push(Data::new_terminal(term.into_term().unwrap()));
+                    self.data_stack.push_terminal(term.into_term().unwrap());
                     self.location_stack.push(location.unwrap());
                     self.precedence_stack.push(shift_prec);
                     self.state_stack
@@ -406,7 +400,7 @@ impl<Data: TokenData, StateIndex: Index + Copy> Context<Data, StateIndex> {
                     Data::Location::new(self.location_stack.iter().rev(), tokens_len);
 
                 // call reduce action
-                let new_data = match Data::reduce_action(
+                match Data::reduce_action(
                     &mut self.data_stack,
                     &mut self.location_stack,
                     reduce_rule,
@@ -415,12 +409,11 @@ impl<Data: TokenData, StateIndex: Index + Copy> Context<Data, StateIndex> {
                     userdata,
                     &mut new_location,
                 ) {
-                    Ok(new_data) => new_data,
+                    Ok(_) => {}
                     Err(err) => {
                         return Err(ParseError::ReduceAction(term, location, err));
                     }
                 };
-                self.data_stack.push(new_data);
                 self.location_stack.push(new_location);
 
                 // construct tree
@@ -463,12 +456,15 @@ impl<Data: TokenData, StateIndex: Index + Copy> Context<Data, StateIndex> {
             self.tree_stack
                 .push(crate::tree::Tree::new_terminal(term.clone()));
 
-            let data = match term {
-                TerminalSymbol::Term(term) => Data::new_terminal(term),
-                TerminalSymbol::Error => Data::new_error(),
-                TerminalSymbol::Eof => Data::new_empty(),
-            };
-            self.data_stack.push(data);
+            match term {
+                TerminalSymbol::Term(term) => {
+                    self.data_stack.push_terminal(term);
+                }
+                TerminalSymbol::Error | TerminalSymbol::Eof => {
+                    self.data_stack.push_empty();
+                }
+            }
+            // location is `Some` if it is not `Eof`
             if let Some(location) = location {
                 self.location_stack.push(location);
             }
@@ -859,13 +855,13 @@ impl<Data: TokenData, StateIndex: Index + Copy> Context<Data, StateIndex> {
     }
 }
 
-impl<Data: TokenData, StateIndex: Index + Copy> Default for Context<Data, StateIndex> {
+impl<Data: DataStack, StateIndex: Index + Copy> Default for Context<Data, StateIndex> {
     fn default() -> Self {
         Self::new()
     }
 }
 
-impl<Data: TokenData, StateIndex: Index + Copy> Clone for Context<Data, StateIndex>
+impl<Data: DataStack, StateIndex: Index + Copy> Clone for Context<Data, StateIndex>
 where
     Data: Clone,
     Data::Term: Clone,
@@ -885,7 +881,7 @@ where
 }
 
 #[cfg(feature = "tree")]
-impl<Data: TokenData, StateIndex: Index + Copy> std::fmt::Display for Context<Data, StateIndex>
+impl<Data: DataStack, StateIndex: Index + Copy> std::fmt::Display for Context<Data, StateIndex>
 where
     Data::Term: std::fmt::Display + Clone,
     Data::NonTerm: std::fmt::Display + Clone + crate::nonterminal::NonTerminal,
@@ -895,7 +891,7 @@ where
     }
 }
 #[cfg(feature = "tree")]
-impl<Data: TokenData, StateIndex: Index + Copy> std::fmt::Debug for Context<Data, StateIndex>
+impl<Data: DataStack, StateIndex: Index + Copy> std::fmt::Debug for Context<Data, StateIndex>
 where
     Data::Term: std::fmt::Debug + Clone,
     Data::NonTerm: std::fmt::Debug + Clone + crate::nonterminal::NonTerminal,
@@ -906,15 +902,15 @@ where
 }
 
 #[cfg(feature = "tree")]
-impl<Data: TokenData, StateIndex: Index + Copy> std::ops::Deref for Context<Data, StateIndex> {
+impl<Data: DataStack, StateIndex: Index + Copy> std::ops::Deref for Context<Data, StateIndex> {
     type Target = crate::tree::TreeList<Data::Term, Data::NonTerm>;
     fn deref(&self) -> &Self::Target {
         &self.tree_stack
     }
 }
 
 #[cfg(feature = "tree")]
-impl<Data: TokenData, StateIndex: Index + Copy> std::ops::DerefMut for Context<Data, StateIndex> {
+impl<Data: DataStack, StateIndex: Index + Copy> std::ops::DerefMut for Context<Data, StateIndex> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         &mut self.tree_stack
     }

rusty_lr_core/src/parser/deterministic/error.rs

@@ -1,12 +1,12 @@
 use std::fmt::Debug;
 use std::fmt::Display;
 
-use crate::nonterminal::TokenData;
+use crate::parser::data_stack::DataStack;
 use crate::TerminalSymbol;
 
 /// Error type for feed()
 #[derive(Clone)]
-pub enum ParseError<Data: TokenData> {
+pub enum ParseError<Data: DataStack> {
     /// No action defined for the given terminal in the parser table.
     /// location will be `None` if the terminal was eof.
     NoAction(TerminalSymbol<Data::Term>, Option<Data::Location>),
@@ -25,7 +25,7 @@ pub enum ParseError<Data: TokenData> {
     NoPrecedence(TerminalSymbol<Data::Term>, Option<Data::Location>, usize),
 }
 
-impl<Data: TokenData> ParseError<Data> {
+impl<Data: DataStack> ParseError<Data> {
     /// location will be `None` if the terminal was eof.
     pub fn location(&self) -> &Option<Data::Location> {
         match self {
@@ -44,7 +44,7 @@ impl<Data: TokenData> ParseError<Data> {
     }
 }
 
-impl<Data: TokenData> Display for ParseError<Data>
+impl<Data: DataStack> Display for ParseError<Data>
 where
     Data::Term: Display,
     Data::ReduceActionError: Display,
@@ -64,7 +64,7 @@ where
     }
 }
 
-impl<Data: TokenData> Debug for ParseError<Data>
+impl<Data: DataStack> Debug for ParseError<Data>
 where
     Data::Term: Debug,
     Data::ReduceActionError: Debug,

rusty_lr_core/src/parser/mod.rs

@@ -4,6 +4,8 @@ pub mod deterministic;
 /// Core parser functionality for non-deterministic parsers
 pub mod nondeterministic;
 
+pub mod data_stack;
+
 pub mod state;
 pub use state::State;