Removed ruleset, dense option

SYNTAX.md

@@ -21,7 +21,6 @@ This document provides a comprehensive guide to the grammar definition syntax us
   - [Operator Precedence (`%left`, `%right`, `%precedence`, `%prec`)](#operator-precedence)
 - [Error Type (`%error`)](#error-type-optional)
 - [Disabling Table Optimization (`%nooptim`)](#no-optimization)
-- [Dense Parser Tables (`%dense`)](#dense-parser-table)
 - [Location Tracking (`%location`)](#location-tracking)
 - [Variable Substitution](#variable-substitution)
 - [Advanced Parser Controls](#advanced-parser-controls)
@@ -449,17 +448,7 @@ Defines a custom error type returned by reduce actions. If your reduce action re
 
 Disables optimization passes on the generated table (which merge states and group equivalent terminals). Use this for debugging or to speed up the parser generation compilation phase itself.
 
----
-
-## Dense Parser Table
 
-```
-%dense;
-```
-
-Forces the generated parser table to use flat `Vec` indexes instead of `HashMap` lookups. This significantly speeds up token feeding, but can drastically increase the binary size of the generated parser.
-
----
 
 ## Location Tracking
 

example/json/src/parser.rs

@@ -5,7 +5,6 @@
 %tokentype char;
 %start Json;
 %userdata Vec<std::ops::Range<usize>>;
-%dense;
 %location std::ops::Range<usize>;
 
 Json: Element;

rusty_lr_buildscript/src/lib.rs

@@ -16,6 +16,7 @@
 //!
 
 pub mod output;
+pub use rusty_lr_parser::TableLayout;
 mod split;
 mod utils;
 
@@ -68,7 +69,8 @@ pub struct Builder {
 
     /// if Some, override the settings with these values
     glr: Option<bool>,
-    dense: Option<bool>,
+    layout: Option<rusty_lr_parser::TableLayout>,
+    dense_limit: Option<usize>,
 }
 
 impl Builder {
@@ -83,7 +85,8 @@ impl Builder {
             is_executable: false,
 
             glr: None,
-            dense: None,
+            layout: None,
+            dense_limit: None,
         }
     }
 
@@ -93,8 +96,24 @@ impl Builder {
         self
     }
     /// override the settings
+    /// Set layout strategy (dense, sparse, auto)
+    pub fn layout(&mut self, layout: rusty_lr_parser::TableLayout) -> &mut Self {
+        self.layout = Some(layout);
+        self
+    }
+    /// Set dense limit in bytes for auto-layout detection
+    pub fn dense_limit(&mut self, limit: usize) -> &mut Self {
+        self.dense_limit = Some(limit);
+        self
+    }
+    /// Deprecated: use layout(TableLayout::Dense) instead
+    #[deprecated(since = "0.66.0", note = "Use `layout(TableLayout::Dense)` instead")]
     pub fn dense(&mut self, dense: bool) -> &mut Self {
-        self.dense = Some(dense);
+        self.layout = Some(if dense {
+            rusty_lr_parser::TableLayout::Dense
+        } else {
+            rusty_lr_parser::TableLayout::Sparse
+        });
         self
     }
 
@@ -576,8 +595,11 @@ impl Builder {
         if let Some(glr) = self.glr {
             grammar_args.glr = glr;
         }
-        if let Some(dense) = self.dense {
-            grammar_args.dense = dense;
+        if let Some(layout) = self.layout {
+            grammar_args.layout = layout;
+        }
+        if let Some(limit) = self.dense_limit {
+            grammar_args.dense_limit = limit;
         }
 
         let span_manager = grammar_args.span_manager.clone();

rusty_lr_core/src/parser/state.rs

@@ -134,9 +134,6 @@ pub trait State {
     /// Get the set of production rule for reduce in this state
     fn expected_reduce_rule(&self) -> impl Iterator<Item = impl Index> + '_;
 
-    /// Get the set of rules that this state is trying to parse
-    fn get_rules(&self) -> &[crate::rule::ShiftedRuleRef];
-
     fn can_accept_error(&self) -> TriState;
 }
 
@@ -152,9 +149,6 @@ pub struct SparseState<TermClass, NonTerm, RuleContainer, StateIndex> {
     /// terminal symbol -> reduce rule index
     pub(crate) reduce_map: HashMap<TermClass, RuleContainer>,
 
-    /// set of rules that this state is trying to parse
-    pub(crate) ruleset: Vec<crate::rule::ShiftedRuleRef>,
-
     pub(crate) can_accept_error: TriState,
 }
 
@@ -193,9 +187,6 @@ impl<
     fn expected_reduce_rule(&self) -> impl Iterator<Item = impl Index> + '_ {
         self.reduce_map.values().flat_map(RuleContainer::to_iter)
     }
-    fn get_rules(&self) -> &[crate::rule::ShiftedRuleRef] {
-        &self.ruleset
-    }
     fn can_accept_error(&self) -> TriState {
         self.can_accept_error
     }
@@ -222,9 +213,6 @@ pub struct DenseState<TermClass, NonTerm, RuleContainer, StateIndex> {
     /// reduce_map[i] will contain i+offset 'th class's reduce rule.
     pub(crate) reduce_offset: usize,
 
-    /// set of rules that this state is trying to parse
-    pub(crate) ruleset: Vec<crate::rule::ShiftedRuleRef>,
-
     pub(crate) can_accept_error: TriState,
 }
 impl<
@@ -274,10 +262,6 @@ impl<
             .flat_map(RuleContainer::to_iter)
     }
 
-    fn get_rules(&self) -> &[crate::rule::ShiftedRuleRef] {
-        &self.ruleset
-    }
-
     fn can_accept_error(&self) -> TriState {
         self.can_accept_error
     }
@@ -326,7 +310,6 @@ where
                     )
                 })
                 .collect(),
-            ruleset: builder_state.ruleset.into_iter().collect(),
             can_accept_error: builder_state.can_accept_error,
         }
     }
@@ -435,7 +418,6 @@ where
             shift_nonterm_offset: nonterm_min,
             reduce_map,
             reduce_offset: reduce_min,
-            ruleset: builder_state.ruleset.into_iter().collect(),
             can_accept_error: builder_state.can_accept_error,
         }
     }

rusty_lr_executable/src/arg.rs

@@ -42,11 +42,32 @@ pub struct Args {
     #[arg(long)]
     pub glr: Option<bool>,
 
-    /// Override the written code and set generated parser table to use dense arrays
-    #[arg(long)]
-    pub dense: Option<bool>,
+    /// Set generated parser table layout representation (auto: optimal choice based on table size, dense: O(1) array, sparse: binary search)
+    #[arg(long, value_enum, default_value_t = TableLayoutArg::Auto)]
+    pub layout: TableLayoutArg,
+
+    /// Set dense limit in bytes for auto-layout detection
+    #[arg(long, default_value_t = 32768)]
+    pub dense_limit: usize,
 
     /// Print the details of a specific state
     #[arg(long)]
     pub state: Option<usize>,
 }
+
+#[derive(clap::ValueEnum, Clone, Copy, Debug, PartialEq, Eq)]
+pub enum TableLayoutArg {
+    Auto,
+    Dense,
+    Sparse,
+}
+
+impl From<TableLayoutArg> for rusty_lr_buildscript::TableLayout {
+    fn from(arg: TableLayoutArg) -> Self {
+        match arg {
+            TableLayoutArg::Auto => rusty_lr_buildscript::TableLayout::Auto,
+            TableLayoutArg::Dense => rusty_lr_buildscript::TableLayout::Dense,
+            TableLayoutArg::Sparse => rusty_lr_buildscript::TableLayout::Sparse,
+        }
+    }
+}

rusty_lr_executable/src/main.rs

@@ -31,9 +31,8 @@ fn main() {
     if let Some(glr) = args.glr {
         builder.glr(glr);
     }
-    if let Some(dense) = args.dense {
-        builder.dense(dense);
-    }
+    builder.layout(args.layout.into());
+    builder.dense_limit(args.dense_limit);
 
     let out = match builder.build_impl() {
         Ok(out) => out,

rusty_lr_parser/src/emit.rs

@@ -718,14 +718,11 @@ impl Grammar {
         let mut shift_nonterm_offsets = Vec::new();
         let mut reduce_data = Vec::new();
         let mut reduce_offsets = Vec::new();
-        let mut ruleset_data = Vec::new();
-        let mut ruleset_offsets = Vec::new();
         let mut can_accept_error = Vec::new();
 
         shift_term_offsets.push(0);
         shift_nonterm_offsets.push(0);
         reduce_offsets.push(0);
-        ruleset_offsets.push(0);
 
         for state in &self.states {
             // 1. shift_goto_map_term
@@ -797,25 +794,6 @@ impl Grammar {
             }
             reduce_offsets.push(reduce_data.len() as u32);
 
-            // 4. ruleset: Vec<ShiftedRuleRef>
-            for &rule in &state.ruleset {
-                let rule_val = rule.rule;
-                let shifted_val = rule.shifted;
-                assert!(
-                    rule_val < 65536,
-                    "Rule index {} in ruleset exceeds 16-bit limit (65536)",
-                    rule_val
-                );
-                assert!(
-                    shifted_val < 65536,
-                    "Shifted index {} in ruleset exceeds 16-bit limit (65536)",
-                    shifted_val
-                );
-                let val = (rule_val as u32) | ((shifted_val as u32) << 16);
-                ruleset_data.push(val);
-            }
-            ruleset_offsets.push(ruleset_data.len() as u32);
-
             // 5. can_accept_error: TriState
             let tri_val = match state.can_accept_error {
                 rusty_lr_core::TriState::False => 0u8,
@@ -829,6 +807,42 @@ impl Grammar {
         let num_states = self.states.len();
         let error_used = self.error_used;
 
+        // to remove suffix from generated data
+        let rule_names = rule_names
+            .into_iter()
+            .map(proc_macro2::Literal::u32_unsuffixed);
+        let rule_precedences = rule_precedences
+            .into_iter()
+            .map(proc_macro2::Literal::u32_unsuffixed);
+        let rule_tokens_data = rule_tokens_data
+            .into_iter()
+            .map(proc_macro2::Literal::u32_unsuffixed);
+        let rule_tokens_offsets = rule_tokens_offsets
+            .into_iter()
+            .map(proc_macro2::Literal::u32_unsuffixed);
+
+        let shift_term_data = shift_term_data
+            .into_iter()
+            .map(proc_macro2::Literal::u32_unsuffixed);
+        let shift_term_offsets = shift_term_offsets
+            .into_iter()
+            .map(proc_macro2::Literal::u32_unsuffixed);
+        let shift_nonterm_data = shift_nonterm_data
+            .into_iter()
+            .map(proc_macro2::Literal::u32_unsuffixed);
+        let shift_nonterm_offsets = shift_nonterm_offsets
+            .into_iter()
+            .map(proc_macro2::Literal::u32_unsuffixed);
+        let reduce_data = reduce_data
+            .into_iter()
+            .map(proc_macro2::Literal::u32_unsuffixed);
+        let reduce_offsets = reduce_offsets
+            .into_iter()
+            .map(proc_macro2::Literal::u32_unsuffixed);
+        let can_accept_error = can_accept_error
+            .into_iter()
+            .map(proc_macro2::Literal::u8_unsuffixed);
+
         // range-compressed Vec based terminal-class_id map
         stream.extend(quote! {
             /// A lightweight parser struct that references the static parser tables and production rules.
@@ -920,17 +934,13 @@ impl Grammar {
                         // - SHIFT_TERM_OFFSETS & SHIFT_NONTERM_OFFSETS: Boundaries separating transitions for each state
                         // - REDUCE_DATA: Variable-length reduce map encoding (term_class, len, rules...)
                         // - REDUCE_OFFSETS: Boundaries separating reduce maps for each state
-                        // - RULESET_DATA: Packed shifted rule references (shifted_idx << 16) | (rule_idx)
-                        // - RULESET_OFFSETS: Boundaries separating ruleset references for each state
                         // - CAN_ACCEPT_ERROR: TriState (0 = False, 1 = True, 2 = Maybe)
                         static SHIFT_TERM_DATA: &[u32] = &[ #(#shift_term_data),* ];
                         static SHIFT_TERM_OFFSETS: &[u32] = &[ #(#shift_term_offsets),* ];
                         static SHIFT_NONTERM_DATA: &[u32] = &[ #(#shift_nonterm_data),* ];
                         static SHIFT_NONTERM_OFFSETS: &[u32] = &[ #(#shift_nonterm_offsets),* ];
                         static REDUCE_DATA: &[u32] = &[ #(#reduce_data),* ];
                         static REDUCE_OFFSETS: &[u32] = &[ #(#reduce_offsets),* ];
-                        static RULESET_DATA: &[u32] = &[ #(#ruleset_data),* ];
-                        static RULESET_OFFSETS: &[u32] = &[ #(#ruleset_offsets),* ];
                         static CAN_ACCEPT_ERROR: &[u8] = &[ #(#can_accept_error),* ];
 
                         let num_states = #num_states;
@@ -977,17 +987,6 @@ impl Grammar {
                                 idx += 2 + len;
                             }
 
-                            // Decode the ruleset containing shifted rule references (rule index, shifted dot index)
-                            let ruleset_start = RULESET_OFFSETS[i] as usize;
-                            let ruleset_end = RULESET_OFFSETS[i + 1] as usize;
-                            let mut ruleset = Vec::with_capacity(ruleset_end - ruleset_start);
-                            for idx in ruleset_start..ruleset_end {
-                                let val = RULESET_DATA[idx];
-                                let rule = (val & 0xffff) as usize;
-                                let shifted = (val >> 16) as usize;
-                                ruleset.push(#module_prefix::rule::ShiftedRuleRef { rule, shifted });
-                            }
-
                             let can_accept_error = match CAN_ACCEPT_ERROR[i] {
                                 0 => #module_prefix::TriState::False,
                                 1 => #module_prefix::TriState::True,
@@ -999,7 +998,7 @@ impl Grammar {
                                 shift_goto_map_term,
                                 shift_goto_map_nonterm,
                                 reduce_map,
-                                ruleset,
+                                ruleset: Vec::new(),
                                 can_accept_error,
                             };
                             states.push(intermediate.into());