Skip to content

Commit 8cd38e8

Browse files
ltrattltratt
authored
Merge pull request #530 from ratmice/bincode
Initial experiment with direct bincode/optional serde.
2 parents ea448ef + 28aa3e8 commit 8cd38e8

File tree

14 files changed

+138
-21
lines changed

14 files changed

+138
-21
lines changed

‎Cargo.toml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ codegen-units = 1
2727
panic = 'unwind'
2828

2929
[workspace.dependencies]
30-
bincode = { version = "2.0", features = ["serde"] }
30+
bincode = "2.0"
3131
cactus = "1.0"
3232
filetime = "0.2"
3333
fnv = "1.0"
@@ -40,10 +40,10 @@ quote = "1.0"
4040
regex = "1.3"
4141
regex-syntax = "0.8"
4242
serde = "1.0"
43-
sparsevec = "0.2"
43+
sparsevec = "0.2.2"
4444
static_assertions = "1.1"
4545
unicode-width = "0.1.11"
46-
vob = ">=3.0.2"
46+
vob = "3.0.4"
4747
proc-macro2 = "1.0"
4848
prettyplease = "0.2.31"
4949
syn = "2.0"

‎cfgrammar/Cargo.toml

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,21 @@ license = "Apache-2.0/MIT"
99
categories = ["parsing"]
1010
keywords = ["yacc", "grammar"]
1111

12+
[features]
13+
serde = ["dep:serde", "serde/derive", "vob/serde"]
14+
bincode = ["dep:bincode", "vob/bincode"]
15+
1216
[lib]
1317
name = "cfgrammar"
1418
path = "src/lib/mod.rs"
1519

1620
[dependencies]
21+
bincode = { workspace = true, optional = true, features = ["derive"] }
1722
lazy_static.workspace = true
1823
indexmap.workspace = true
1924
num-traits.workspace = true
2025
regex.workspace = true
21-
serde = { workspace = true, features = ["derive"], optional = true }
22-
vob = { workspace = true, features = ["serde"] }
26+
serde = { workspace = true, optional = true }
27+
vob = { workspace = true }
2328
quote.workspace = true
2429
proc-macro2.workspace = true

‎cfgrammar/src/lib/idxnewtype.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33

44
use std::mem::size_of;
55

6+
#[cfg(feature = "bincode")]
7+
use bincode::{Decode, Encode};
68
use num_traits::{PrimInt, Unsigned};
79
#[cfg(feature = "serde")]
810
use serde::{Deserialize, Serialize};
@@ -12,6 +14,7 @@ macro_rules! IdxNewtype {
1214
$(#[$attr])*
1315
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
1416
#[cfg_attr(feature="serde", derive(Serialize, Deserialize))]
17+
#[cfg_attr(feature="bincode", derive(Encode, Decode))]
1518
pub struct $n<T>(pub T);
1619

1720
impl<T: PrimInt + Unsigned> From<$n<T>> for usize {

‎cfgrammar/src/lib/mod.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@
5151
//! [`YaccGrammar::new_with_storaget()`](yacc/grammar/struct.YaccGrammar.html#method.new_with_storaget)
5252
//! which take as input a Yacc grammar.
5353
54+
#[cfg(feature = "bincode")]
55+
use bincode::{Decode, Encode};
5456
#[cfg(feature = "serde")]
5557
use serde::{Deserialize, Serialize};
5658

@@ -67,6 +69,7 @@ pub use crate::idxnewtype::{PIdx, RIdx, SIdx, TIdx};
6769

6870
#[derive(Clone, Copy, Debug, Hash, Eq, PartialEq)]
6971
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
72+
#[cfg_attr(feature = "bincode", derive(Encode, Decode))]
7073
pub enum Symbol<StorageT> {
7174
Rule(RIdx<StorageT>),
7275
Token(TIdx<StorageT>),

‎cfgrammar/src/lib/span.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#[cfg(feature = "bincode")]
2+
use bincode::{Decode, Encode};
13
use proc_macro2::TokenStream;
24
use quote::{quote, ToTokens, TokenStreamExt};
35
#[cfg(feature = "serde")]
@@ -7,6 +9,7 @@ use serde::{Deserialize, Serialize};
79
/// references (i.e. the `Span` doesn't hold a reference / copy of the actual input).
810
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
911
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
12+
#[cfg_attr(feature = "bincode", derive(Encode, Decode))]
1013
pub struct Span {
1114
start: usize,
1215
end: usize,

‎cfgrammar/src/lib/yacc/grammar.rs

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
#![allow(clippy::derive_partial_eq_without_eq)]
22
use std::{cell::RefCell, collections::HashMap, fmt::Write};
33

4+
#[cfg(feature = "bincode")]
5+
use bincode::{Decode, Encode};
46
use num_traits::{AsPrimitive, PrimInt, Unsigned};
57
#[cfg(feature = "serde")]
68
use serde::{Deserialize, Serialize};
@@ -19,13 +21,15 @@ const IMPLICIT_START_RULE: &str = "^~";
1921
pub type PrecedenceLevel = u64;
2022
#[derive(Clone, Copy, Debug, PartialEq)]
2123
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
24+
#[cfg_attr(feature = "bincode", derive(Encode, Decode))]
2225
pub struct Precedence {
2326
pub level: PrecedenceLevel,
2427
pub kind: AssocKind,
2528
}
2629

2730
#[derive(Clone, Copy, Debug, PartialEq)]
2831
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
32+
#[cfg_attr(feature = "bincode", derive(Encode, Decode))]
2933
pub enum AssocKind {
3034
Left,
3135
Right,
@@ -35,6 +39,7 @@ pub enum AssocKind {
3539
/// Representation of a `YaccGrammar`. See the [top-level documentation](../../index.html) for the
3640
/// guarantees this struct makes about rules, tokens, productions, and symbols.
3741
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
42+
#[cfg_attr(feature = "bincode", derive(Encode))]
3843
pub struct YaccGrammar<StorageT = u32> {
3944
/// How many rules does this grammar have?
4045
rules_len: RIdx<StorageT>,
@@ -88,6 +93,88 @@ pub struct YaccGrammar<StorageT = u32> {
8893
expectrr: Option<usize>,
8994
}
9095

96+
// The implementations of `Decode` and `BorrowDecode`
97+
// Contain a modified copy of the output of `cargo expand` of the derivation:
98+
// #[cfg_attr(feature = "bincode", derive(Decode))]
99+
//
100+
// The current version of bincode has bugs in it's derive macro implementation
101+
// https://github.com/bincode-org/bincode/issues/763
102+
// https://github.com/bincode-org/bincode/issues/646
103+
//
104+
// Once those are fixed, we can replace this with derive.
105+
106+
#[cfg(feature = "bincode")]
107+
impl<StorageT, __Context> Decode<__Context> for YaccGrammar<StorageT>
108+
where
109+
StorageT: Decode<__Context> + 'static,
110+
{
111+
fn decode<__D: bincode::de::Decoder<Context = __Context>>(
112+
decoder: &mut __D,
113+
) -> Result<Self, bincode::error::DecodeError> {
114+
Ok(Self {
115+
rules_len: Decode::decode(decoder)?,
116+
rule_names: Decode::decode(decoder)?,
117+
token_names: Decode::decode(decoder)?,
118+
token_precs: Decode::decode(decoder)?,
119+
token_epp: Decode::decode(decoder)?,
120+
tokens_len: Decode::decode(decoder)?,
121+
eof_token_idx: Decode::decode(decoder)?,
122+
prods_len: Decode::decode(decoder)?,
123+
start_prod: Decode::decode(decoder)?,
124+
prods: Decode::decode(decoder)?,
125+
rules_prods: Decode::decode(decoder)?,
126+
prods_rules: Decode::decode(decoder)?,
127+
prod_precs: Decode::decode(decoder)?,
128+
implicit_rule: Decode::decode(decoder)?,
129+
actions: Decode::decode(decoder)?,
130+
parse_param: Decode::decode(decoder)?,
131+
programs: Decode::decode(decoder)?,
132+
actiontypes: Decode::decode(decoder)?,
133+
avoid_insert: Decode::decode(decoder)?,
134+
expect: Decode::decode(decoder)?,
135+
expectrr: Decode::decode(decoder)?,
136+
})
137+
}
138+
}
139+
140+
// Eventually this should just be provided through:
141+
// #[cfg_attr(feature = "bincode", derive(Decode))]
142+
//
143+
// See comment at the corresponding bincode::Decode impl.
144+
#[cfg(feature = "bincode")]
145+
impl<'__de, StorageT, __Context> bincode::BorrowDecode<'__de, __Context> for YaccGrammar<StorageT>
146+
where
147+
StorageT: bincode::de::BorrowDecode<'__de, __Context> + '__de,
148+
{
149+
fn borrow_decode<__D: ::bincode::de::BorrowDecoder<'__de, Context = __Context>>(
150+
decoder: &mut __D,
151+
) -> Result<Self, bincode::error::DecodeError> {
152+
Ok(Self {
153+
rules_len: bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
154+
rule_names: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
155+
token_names: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
156+
token_precs: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
157+
token_epp: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
158+
tokens_len: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
159+
eof_token_idx: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
160+
prods_len: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
161+
start_prod: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
162+
prods: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
163+
rules_prods: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
164+
prods_rules: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
165+
prod_precs: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
166+
implicit_rule: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
167+
actions: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
168+
parse_param: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
169+
programs: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
170+
actiontypes: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
171+
avoid_insert: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
172+
expect: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
173+
expectrr: ::bincode::BorrowDecode::<'_, __Context>::borrow_decode(decoder)?,
174+
})
175+
}
176+
}
177+
91178
// Internally, we assume that a grammar's start rule has a single production. Since we manually
92179
// create the start rule ourselves (without relying on user input), this is a safe assumption.
93180

‎cfgrammar/src/lib/yacc/parser.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
// Note: this is the parser for both YaccKind::Original(YaccOriginalActionKind::GenericParseTree) and YaccKind::Eco yacc kinds.
22

3+
#[cfg(feature = "bincode")]
4+
use bincode::{Decode, Encode};
35
use lazy_static::lazy_static;
46
use num_traits::PrimInt;
57
use regex::Regex;
@@ -160,6 +162,7 @@ impl fmt::Display for YaccGrammarErrorKind {
160162
/// The various different possible Yacc parser errors.
161163
#[derive(Debug, PartialEq, Eq, Clone)]
162164
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
165+
#[cfg_attr(feature = "bincode", derive(Encode, Decode))]
163166
#[non_exhaustive]
164167
pub enum YaccGrammarWarningKind {
165168
UnusedRule,
@@ -169,6 +172,7 @@ pub enum YaccGrammarWarningKind {
169172
/// Any Warning from the Yacc parser returns an instance of this struct.
170173
#[derive(Debug, PartialEq, Eq, Clone)]
171174
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
175+
#[cfg_attr(feature = "bincode", derive(Encode, Decode))]
172176
pub struct YaccGrammarWarning {
173177
/// The specific kind of warning.
174178
pub(crate) kind: YaccGrammarWarningKind,

‎lrlex/Cargo.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ regex-syntax.workspace = true
3434
num-traits.workspace = true
3535
proc-macro2.workspace = true
3636
quote.workspace = true
37-
serde.workspace = true
37+
bincode.workspace = true
38+
serde = { workspace = true, optional = true }
3839
prettyplease.workspace = true
3940
syn.workspace = true

‎lrlex/src/lib/ctbuilder.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,14 @@ use std::{
1414
sync::Mutex,
1515
};
1616

17+
use bincode::Encode;
1718
use cfgrammar::{newlinecache::NewlineCache, Spanned};
1819
use lazy_static::lazy_static;
1920
use lrpar::{CTParserBuilder, LexerTypes};
2021
use num_traits::{AsPrimitive, PrimInt, Unsigned};
2122
use proc_macro2::TokenStream;
2223
use quote::{format_ident, quote, ToTokens, TokenStreamExt};
2324
use regex::Regex;
24-
use serde::Serialize;
2525

2626
use crate::{DefaultLexerTypes, LRNonStreamingLexerDef, LexFlags, LexerDef, UNSPECIFIED_LEX_FLAGS};
2727

@@ -150,7 +150,7 @@ impl CTLexerBuilder<'_, DefaultLexerTypes<u32>> {
150150
impl<'a, LexerTypesT: LexerTypes> CTLexerBuilder<'a, LexerTypesT>
151151
where
152152
LexerTypesT::StorageT:
153-
'static + Debug + Eq + Hash + PrimInt + Serialize + TryFrom<usize> + Unsigned + ToTokens,
153+
'static + Debug + Eq + Hash + PrimInt + Encode + TryFrom<usize> + Unsigned + ToTokens,
154154
usize: AsPrimitive<LexerTypesT::StorageT>,
155155
{
156156
/// Create a new [CTLexerBuilder].

‎lrpar/Cargo.toml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,18 @@ name = "lrpar"
1515
path = "src/lib/mod.rs"
1616

1717
[features]
18-
serde = []
18+
serde = ["dep:serde", "cfgrammar/serde", "lrtable/serde"]
1919
_unstable_api = []
2020
_unsealed_unstable_traits = ["_unstable_api"]
2121

2222
[build-dependencies]
2323
vergen = { version = "8", default-features = false, features = ["build"] }
2424

2525
[dependencies]
26-
cfgrammar = { path="../cfgrammar", version = "0.13", features=["serde"] }
27-
lrtable = { path="../lrtable", version = "0.13", features=["serde"] }
26+
cfgrammar = { path="../cfgrammar", version = "0.13", features = ["bincode"] }
27+
lrtable = { path="../lrtable", version = "0.13", features = ["bincode"] }
2828

29-
bincode.workspace = true
29+
bincode = { workspace = true, features = ["derive"] }
3030
cactus.workspace = true
3131
filetime.workspace = true
3232
indexmap.workspace = true
@@ -36,7 +36,7 @@ packedvec.workspace = true
3636
proc-macro2.workspace = true
3737
quote.workspace = true
3838
regex.workspace = true
39-
serde = { workspace = true, features = ["derive"] }
39+
serde = { workspace = true, features = ["derive"], optional = true }
4040
static_assertions.workspace = true
4141
vob.workspace = true
4242
syn.workspace = true

‎lrpar/src/lib/ctbuilder.rs

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ use std::{
1515
};
1616

1717
use crate::{LexerTypes, RecoveryKind};
18-
use bincode::serde::{decode_from_slice, encode_to_vec};
18+
use bincode::{decode_from_slice, encode_to_vec, Decode, Encode};
1919
use cfgrammar::{
2020
newlinecache::NewlineCache,
2121
yacc::{
@@ -30,7 +30,6 @@ use num_traits::{AsPrimitive, PrimInt, Unsigned};
3030
use proc_macro2::{Literal, TokenStream};
3131
use quote::{format_ident, quote, ToTokens, TokenStreamExt};
3232
use regex::Regex;
33-
use serde::{de::DeserializeOwned, Serialize};
3433

3534
const ACTION_PREFIX: &str = "__gt_";
3635
const GLOBAL_PREFIX: &str = "__GT_";
@@ -248,7 +247,7 @@ where
248247

249248
impl<
250249
'a,
251-
StorageT: 'static + Debug + Hash + PrimInt + Serialize + Unsigned,
250+
StorageT: 'static + Debug + Hash + PrimInt + Encode + Unsigned,
252251
LexerTypesT: LexerTypes<StorageT = StorageT>,
253252
> CTParserBuilder<'a, LexerTypesT>
254253
where
@@ -1335,9 +1334,9 @@ where
13351334
}
13361335

13371336
/// This function is called by generated files; it exists so that generated files don't require a
1338-
/// dependency on serde and rmps.
1337+
/// direct dependency on bincode.
13391338
#[doc(hidden)]
1340-
pub fn _reconstitute<StorageT: DeserializeOwned + Hash + PrimInt + Unsigned>(
1339+
pub fn _reconstitute<StorageT: Decode<()> + Hash + PrimInt + Unsigned + 'static>(
13411340
grm_buf: &[u8],
13421341
stable_buf: &[u8],
13431342
) -> (YaccGrammar<StorageT>, StateTable<StorageT>) {

‎lrtable/Cargo.toml

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,20 @@ readme = "README.md"
88
license = "Apache-2.0/MIT"
99
categories = ["parsing"]
1010

11+
[features]
12+
bincode = ["dep:bincode", "sparsevec/bincode", "cfgrammar/bincode"]
13+
serde = ["dep:serde", "sparsevec/serde", "cfgrammar/serde"]
14+
1115
[lib]
1216
name = "lrtable"
1317
path = "src/lib/mod.rs"
1418

1519
[dependencies]
16-
cfgrammar = { path="../cfgrammar", version = "0.13", features=["serde"] }
20+
cfgrammar = { path="../cfgrammar", version = "0.13" }
1721

22+
bincode = { workspace = true, features = ["derive"], optional = true }
1823
fnv.workspace = true
1924
num-traits.workspace = true
2025
serde = { workspace = true, features = ["derive"], optional = true }
21-
vob = { workspace = true, features = ["serde"] }
22-
sparsevec = { workspace = true, features = ["serde"] }
26+
vob.workspace = true
27+
sparsevec.workspace = true

‎lrtable/src/lib/mod.rs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
use std::{hash::Hash, mem::size_of};
88

9+
#[cfg(feature = "bincode")]
10+
use bincode::{Decode, Encode};
911
use num_traits::{AsPrimitive, PrimInt, Unsigned};
1012
#[cfg(feature = "serde")]
1113
use serde::{Deserialize, Serialize};
@@ -26,6 +28,7 @@ macro_rules! IdxNewtype {
2628
$(#[$attr])*
2729
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
2830
#[cfg_attr(feature="serde", derive(Serialize, Deserialize))]
31+
#[cfg_attr(feature="bincode", derive(Encode, Decode))]
2932
pub struct $n<T>(pub T);
3033

3134
impl<T: PrimInt + Unsigned> From<$n<T>> for usize {

0 commit comments

Comments
 (0)